{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 1, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 2, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 3, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 4, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 5, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 6, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 7, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    }
  ],
  "cupti_version": 22,
  "cuda_runtime_version": 12040,
  "cuda_driver_version": 12080,
  "distributedInfo": {"backend": "nccl", "rank": 7, "world_size": 8, "pg_count": 1, "pg_config": [{"pg_name": "0", "pg_desc": "default_pg", "backend_config": "cuda:nccl", "pg_size": 8, "ranks": [0, 1, 2, 3, 4, 5, 6, 7]}], "nccl_version": "2.21.5"},
  "record_shapes": 1,
  "trace_id": "F8CC8AD19E684984B6873030B96CDC7C",
  "traceEvents": [
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: DivBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706439.840, "dur": 168.312,
    "args": {
      "External id": 2483201,"Record function id": 0, "Sequence number": 24740650, "Fwd thread id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "DivBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706461.914, "dur": 133.508,
    "args": {
      "External id": 2483202,"Sequence number": 24740650, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 1
    }
  },
  {
    "ph": "f", "id": 1, "pid": 1336760, "tid": 1381209, "ts": 1514656706461.914,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706472.030, "dur": 120.481,
    "args": {
      "External id": 2483203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706623.203, "dur": 273.068,
    "args": {
      "External id": 2483204,"Record function id": 0, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706698.641, "dur": 113.053,
    "args": {
      "External id": 2483205,"Record function id": 0, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.27", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706740.564, "dur": 59.190,
    "args": {
      "External id": 2483206,"Record function id": 0, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706816.820, "dur": 2.266,
    "args": {
      "External id": 2483207,"Sequence number": 24740649, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 6
    }
  },
  {
    "ph": "f", "id": 2, "pid": 1336760, "tid": 1381209, "ts": 1514656706816.820,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706824.806, "dur": 64.201,
    "args": {
      "External id": 2483208,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706835.588, "dur": 52.907,
    "args": {
      "External id": 2483209,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706847.291, "dur": 3.980,
    "args": {
      "External id": 2483210,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706905.809, "dur": 36750.504,
    "args": {
      "External id": 2483211,"Record function id": 0, "Sequence number": 24740647, "Fwd thread id": 1, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706907.603, "dur": 36735.901,
    "args": {
      "External id": 2483212,"Sequence number": 24740647, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11
    }
  },
  {
    "ph": "f", "id": 3, "pid": 1336760, "tid": 1381209, "ts": 1514656706907.603,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706962.299, "dur": 4.714,
    "args": {
      "External id": 2483213,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706973.396, "dur": 36524.330,
    "args": {
      "External id": 2483214,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706975.887, "dur": 36521.411,
    "args": {
      "External id": 2483215,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656706981.787, "dur": 43.075,
    "args": {
      "External id": 2483216,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656707028.098, "dur": 36467.018,
    "args": {
      "External id": 2483217,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743503.449, "dur": 0.810,
    "args": {
      "External id": 2483218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743507.544, "dur": 3.781,
    "args": {
      "External id": 2483219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743509.418, "dur": 1.681,
    "args": {
      "External id": 2483220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743518.941, "dur": 47.627,
    "args": {
      "External id": 2483221,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743578.477, "dur": 53.902,
    "args": {
      "External id": 2483222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743580.882, "dur": 51.295,
    "args": {
      "External id": 2483223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743583.153, "dur": 48.658,
    "args": {
      "External id": 2483224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743670.409, "dur": 22.840,
    "args": {
      "External id": 2483225,"Record function id": 0, "Sequence number": 24740646, "Fwd thread id": 1, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743672.625, "dur": 16.796,
    "args": {
      "External id": 2483226,"Sequence number": 24740646, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 25
    }
  },
  {
    "ph": "f", "id": 4, "pid": 1336760, "tid": 1381209, "ts": 1514656743672.625,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743677.418, "dur": 11.758,
    "args": {
      "External id": 2483227,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743681.552, "dur": 7.432,
    "args": {
      "External id": 2483228,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743698.164, "dur": 114.583,
    "args": {
      "External id": 2483229,"Record function id": 0, "Sequence number": 24740645, "Fwd thread id": 1, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743699.325, "dur": 104.360,
    "args": {
      "External id": 2483230,"Sequence number": 24740645, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 29
    }
  },
  {
    "ph": "f", "id": 5, "pid": 1336760, "tid": 1381209, "ts": 1514656743699.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743703.021, "dur": 100.092,
    "args": {
      "External id": 2483231,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743710.085, "dur": 36.636,
    "args": {
      "External id": 2483232,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743714.976, "dur": 6.259,
    "args": {
      "External id": 2483233,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743723.681, "dur": 22.734,
    "args": {
      "External id": 2483234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743727.441, "dur": 18.588,
    "args": {
      "External id": 2483235,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743752.318, "dur": 5.122,
    "args": {
      "External id": 2483236,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743755.692, "dur": 1.375,
    "args": {
      "External id": 2483237,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743758.564, "dur": 43.663,
    "args": {
      "External id": 2483238,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743817.844, "dur": 74.173,
    "args": {
      "External id": 2483239,"Record function id": 0, "Sequence number": 24740644, "Fwd thread id": 1, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743819.024, "dur": 68.138,
    "args": {
      "External id": 2483240,"Sequence number": 24740644, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 39
    }
  },
  {
    "ph": "f", "id": 6, "pid": 1336760, "tid": 1381209, "ts": 1514656743819.024,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743822.880, "dur": 64.032,
    "args": {
      "External id": 2483241,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "3"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743829.128, "dur": 28.015,
    "args": {
      "External id": 2483242,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743830.555, "dur": 3.694,
    "args": {
      "External id": 2483243,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743835.015, "dur": 21.877,
    "args": {
      "External id": 2483244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743838.754, "dur": 17.784,
    "args": {
      "External id": 2483245,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743861.009, "dur": 6.438,
    "args": {
      "External id": 2483246,"Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743865.508, "dur": 1.019,
    "args": {
      "External id": 2483247,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743868.009, "dur": 18.445,
    "args": {
      "External id": 2483248,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743896.165, "dur": 168.462,
    "args": {
      "External id": 2483249,"Record function id": 0, "Sequence number": 24740643, "Fwd thread id": 1, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743897.355, "dur": 162.358,
    "args": {
      "External id": 2483250,"Sequence number": 24740643, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 49
    }
  },
  {
    "ph": "f", "id": 7, "pid": 1336760, "tid": 1381209, "ts": 1514656743897.355,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743902.505, "dur": 156.865,
    "args": {
      "External id": 2483251,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743905.980, "dur": 16.366,
    "args": {
      "External id": 2483252,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743907.276, "dur": 2.534,
    "args": {
      "External id": 2483253,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743910.278, "dur": 11.813,
    "args": {
      "External id": 2483254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743911.250, "dur": 10.382,
    "args": {
      "External id": 2483255,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743925.435, "dur": 2.126,
    "args": {
      "External id": 2483256,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743926.406, "dur": 0.998,
    "args": {
      "External id": 2483257,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656743928.093, "dur": 129.993,
    "args": {
      "External id": 2483258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744072.753, "dur": 103.004,
    "args": {
      "External id": 2483259,"Record function id": 0, "Sequence number": 24740642, "Fwd thread id": 1, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744074.087, "dur": 98.120,
    "args": {
      "External id": 2483260,"Sequence number": 24740642, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 59
    }
  },
  {
    "ph": "f", "id": 8, "pid": 1336760, "tid": 1381209, "ts": 1514656744074.087,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744075.872, "dur": 96.095,
    "args": {
      "External id": 2483261,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744079.181, "dur": 18.690,
    "args": {
      "External id": 2483262,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744080.192, "dur": 4.832,
    "args": {
      "External id": 2483263,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744085.782, "dur": 11.850,
    "args": {
      "External id": 2483264,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744087.041, "dur": 10.159,
    "args": {
      "External id": 2483265,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744098.843, "dur": 4.044,
    "args": {
      "External id": 2483266,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744102.178, "dur": 0.506,
    "args": {
      "External id": 2483267,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744105.333, "dur": 65.929,
    "args": {
      "External id": 2483268,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744180.403, "dur": 36.302,
    "args": {
      "External id": 2483269,"Record function id": 0, "Sequence number": 24740641, "Fwd thread id": 1, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744181.934, "dur": 1.175,
    "args": {
      "External id": 2483270,"Sequence number": 24740641, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 69
    }
  },
  {
    "ph": "f", "id": 9, "pid": 1336760, "tid": 1381209, "ts": 1514656744181.934,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744185.907, "dur": 27.669,
    "args": {
      "External id": 2483271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744188.033, "dur": 25.040,
    "args": {
      "External id": 2483272,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744196.716, "dur": 0.535,
    "args": {
      "External id": 2483273,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744221.762, "dur": 2209.240,
    "args": {
      "External id": 2483274,"Record function id": 0, "Sequence number": 24740639, "Fwd thread id": 1, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744223.460, "dur": 2170.192,
    "args": {
      "External id": 2483275,"Sequence number": 24740639, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 74
    }
  },
  {
    "ph": "f", "id": 10, "pid": 1336760, "tid": 1381209, "ts": 1514656744223.460,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744265.643, "dur": 3.863,
    "args": {
      "External id": 2483276,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744271.999, "dur": 2029.501,
    "args": {
      "External id": 2483277,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744273.415, "dur": 2027.795,
    "args": {
      "External id": 2483278,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744276.176, "dur": 4.634,
    "args": {
      "External id": 2483279,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656744282.015, "dur": 2018.240,
    "args": {
      "External id": 2483280,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746305.315, "dur": 0.482,
    "args": {
      "External id": 2483281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746309.799, "dur": 2.451,
    "args": {
      "External id": 2483282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746311.056, "dur": 0.866,
    "args": {
      "External id": 2483283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746316.849, "dur": 22.127,
    "args": {
      "External id": 2483284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746344.934, "dur": 41.065,
    "args": {
      "External id": 2483285,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746346.295, "dur": 39.528,
    "args": {
      "External id": 2483286,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746347.837, "dur": 37.575,
    "args": {
      "External id": 2483287,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746404.231, "dur": 21.686,
    "args": {
      "External id": 2483288,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746438.183, "dur": 15.499,
    "args": {
      "External id": 2483289,"Record function id": 0, "Sequence number": 24740638, "Fwd thread id": 1, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746439.595, "dur": 11.857,
    "args": {
      "External id": 2483290,"Sequence number": 24740638, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 89
    }
  },
  {
    "ph": "f", "id": 11, "pid": 1336760, "tid": 1381209, "ts": 1514656746439.595,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746446.349, "dur": 4.910,
    "args": {
      "External id": 2483291,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746447.515, "dur": 3.593,
    "args": {
      "External id": 2483292,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746457.130, "dur": 65.838,
    "args": {
      "External id": 2483293,"Record function id": 0, "Sequence number": 24740637, "Fwd thread id": 1, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746458.259, "dur": 59.042,
    "args": {
      "External id": 2483294,"Sequence number": 24740637, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 93
    }
  },
  {
    "ph": "f", "id": 12, "pid": 1336760, "tid": 1381209, "ts": 1514656746458.259,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746460.246, "dur": 56.657,
    "args": {
      "External id": 2483295,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746464.426, "dur": 18.784,
    "args": {
      "External id": 2483296,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746466.050, "dur": 2.757,
    "args": {
      "External id": 2483297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746469.533, "dur": 13.421,
    "args": {
      "External id": 2483298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746471.190, "dur": 11.404,
    "args": {
      "External id": 2483299,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746484.702, "dur": 4.692,
    "args": {
      "External id": 2483300,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746486.364, "dur": 2.786,
    "args": {
      "External id": 2483301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746490.314, "dur": 25.809,
    "args": {
      "External id": 2483302,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746541.626, "dur": 61.055,
    "args": {
      "External id": 2483303,"Record function id": 0, "Sequence number": 24740636, "Fwd thread id": 1, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746543.365, "dur": 54.613,
    "args": {
      "External id": 2483304,"Sequence number": 24740636, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 103
    }
  },
  {
    "ph": "f", "id": 13, "pid": 1336760, "tid": 1381209, "ts": 1514656746543.365,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746548.135, "dur": 49.419,
    "args": {
      "External id": 2483305,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "2"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746550.309, "dur": 22.004,
    "args": {
      "External id": 2483306,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746551.397, "dur": 3.441,
    "args": {
      "External id": 2483307,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746555.547, "dur": 16.491,
    "args": {
      "External id": 2483308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746556.362, "dur": 15.172,
    "args": {
      "External id": 2483309,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746573.608, "dur": 7.217,
    "args": {
      "External id": 2483310,"Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746579.036, "dur": 1.226,
    "args": {
      "External id": 2483311,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746581.447, "dur": 15.525,
    "args": {
      "External id": 2483312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746606.955, "dur": 90.889,
    "args": {
      "External id": 2483313,"Record function id": 0, "Sequence number": 24740635, "Fwd thread id": 1, "Ev Idx": 112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746610.534, "dur": 84.133,
    "args": {
      "External id": 2483314,"Sequence number": 24740635, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 113
    }
  },
  {
    "ph": "f", "id": 14, "pid": 1336760, "tid": 1381209, "ts": 1514656746610.534,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746612.316, "dur": 81.998,
    "args": {
      "External id": 2483315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746613.192, "dur": 15.961,
    "args": {
      "External id": 2483316,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746614.121, "dur": 2.224,
    "args": {
      "External id": 2483317,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746616.932, "dur": 11.980,
    "args": {
      "External id": 2483318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746617.753, "dur": 10.751,
    "args": {
      "External id": 2483319,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746630.205, "dur": 4.784,
    "args": {
      "External id": 2483320,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746633.998, "dur": 0.734,
    "args": {
      "External id": 2483321,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746635.771, "dur": 57.751,
    "args": {
      "External id": 2483322,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746701.777, "dur": 89.024,
    "args": {
      "External id": 2483323,"Record function id": 0, "Sequence number": 24740634, "Fwd thread id": 1, "Ev Idx": 122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746705.053, "dur": 68.406,
    "args": {
      "External id": 2483324,"Sequence number": 24740634, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 123
    }
  },
  {
    "ph": "f", "id": 15, "pid": 1336760, "tid": 1381209, "ts": 1514656746705.053,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746708.386, "dur": 64.775,
    "args": {
      "External id": 2483325,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746709.114, "dur": 14.119,
    "args": {
      "External id": 2483326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746709.670, "dur": 1.644,
    "args": {
      "External id": 2483327,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746711.931, "dur": 11.062,
    "args": {
      "External id": 2483328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746712.961, "dur": 9.715,
    "args": {
      "External id": 2483329,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746724.110, "dur": 2.132,
    "args": {
      "External id": 2483330,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746725.510, "dur": 0.613,
    "args": {
      "External id": 2483331,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746726.711, "dur": 45.805,
    "args": {
      "External id": 2483332,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746777.103, "dur": 12.504,
    "args": {
      "External id": 2483333,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746795.213, "dur": 36.898,
    "args": {
      "External id": 2483334,"Record function id": 0, "Sequence number": 24740633, "Fwd thread id": 1, "Ev Idx": 133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746798.333, "dur": 0.905,
    "args": {
      "External id": 2483335,"Sequence number": 24740633, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 134
    }
  },
  {
    "ph": "f", "id": 16, "pid": 1336760, "tid": 1381209, "ts": 1514656746798.333,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746801.643, "dur": 26.251,
    "args": {
      "External id": 2483336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746803.332, "dur": 24.155,
    "args": {
      "External id": 2483337,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746809.655, "dur": 2.633,
    "args": {
      "External id": 2483338,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746837.243, "dur": 3092.067,
    "args": {
      "External id": 2483339,"Record function id": 0, "Sequence number": 24740631, "Fwd thread id": 1, "Ev Idx": 138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746840.749, "dur": 3064.011,
    "args": {
      "External id": 2483340,"Sequence number": 24740631, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 139
    }
  },
  {
    "ph": "f", "id": 17, "pid": 1336760, "tid": 1381209, "ts": 1514656746840.749,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746871.925, "dur": 2.201,
    "args": {
      "External id": 2483341,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746876.335, "dur": 2939.494,
    "args": {
      "External id": 2483342,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746878.036, "dur": 2937.452,
    "args": {
      "External id": 2483343,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746880.381, "dur": 2.947,
    "args": {
      "External id": 2483344,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656746884.137, "dur": 2930.449,
    "args": {
      "External id": 2483345,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749819.218, "dur": 0.388,
    "args": {
      "External id": 2483346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749821.193, "dur": 2.224,
    "args": {
      "External id": 2483347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749822.604, "dur": 0.686,
    "args": {
      "External id": 2483348,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749826.780, "dur": 21.146,
    "args": {
      "External id": 2483349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749857.135, "dur": 40.688,
    "args": {
      "External id": 2483350,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749858.285, "dur": 39.375,
    "args": {
      "External id": 2483351,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749859.486, "dur": 37.876,
    "args": {
      "External id": 2483352,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749912.843, "dur": 12.985,
    "args": {
      "External id": 2483353,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749939.380, "dur": 11.665,
    "args": {
      "External id": 2483354,"Record function id": 0, "Sequence number": 24740630, "Fwd thread id": 1, "Ev Idx": 153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749940.884, "dur": 6.617,
    "args": {
      "External id": 2483355,"Sequence number": 24740630, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 154
    }
  },
  {
    "ph": "f", "id": 18, "pid": 1336760, "tid": 1381209, "ts": 1514656749940.884,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749943.115, "dur": 4.184,
    "args": {
      "External id": 2483356,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749944.179, "dur": 2.989,
    "args": {
      "External id": 2483357,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749954.223, "dur": 99.089,
    "args": {
      "External id": 2483358,"Record function id": 0, "Sequence number": 24740629, "Fwd thread id": 1, "Ev Idx": 157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749955.452, "dur": 90.194,
    "args": {
      "External id": 2483359,"Sequence number": 24740629, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 158
    }
  },
  {
    "ph": "f", "id": 19, "pid": 1336760, "tid": 1381209, "ts": 1514656749955.452,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749958.637, "dur": 86.690,
    "args": {
      "External id": 2483360,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749962.910, "dur": 44.986,
    "args": {
      "External id": 2483361,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749964.301, "dur": 2.238,
    "args": {
      "External id": 2483362,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749967.483, "dur": 40.097,
    "args": {
      "External id": 2483363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656749968.881, "dur": 37.888,
    "args": {
      "External id": 2483364,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750009.983, "dur": 5.316,
    "args": {
      "External id": 2483365,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750013.952, "dur": 1.075,
    "args": {
      "External id": 2483366,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750016.466, "dur": 28.022,
    "args": {
      "External id": 2483367,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750059.184, "dur": 51.720,
    "args": {
      "External id": 2483368,"Record function id": 0, "Sequence number": 24740628, "Fwd thread id": 1, "Ev Idx": 167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750060.320, "dur": 46.849,
    "args": {
      "External id": 2483369,"Sequence number": 24740628, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 168
    }
  },
  {
    "ph": "f", "id": 20, "pid": 1336760, "tid": 1381209, "ts": 1514656750060.320,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750062.227, "dur": 44.590,
    "args": {
      "External id": 2483370,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750063.952, "dur": 19.026,
    "args": {
      "External id": 2483371,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750067.178, "dur": 2.680,
    "args": {
      "External id": 2483372,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750070.692, "dur": 12.042,
    "args": {
      "External id": 2483373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750071.418, "dur": 10.940,
    "args": {
      "External id": 2483374,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750083.913, "dur": 8.474,
    "args": {
      "External id": 2483375,"Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750088.924, "dur": 3.037,
    "args": {
      "External id": 2483376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750093.114, "dur": 13.190,
    "args": {
      "External id": 2483377,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750114.632, "dur": 90.137,
    "args": {
      "External id": 2483378,"Record function id": 0, "Sequence number": 24740627, "Fwd thread id": 1, "Ev Idx": 177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750115.663, "dur": 86.372,
    "args": {
      "External id": 2483379,"Sequence number": 24740627, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 178
    }
  },
  {
    "ph": "f", "id": 21, "pid": 1336760, "tid": 1381209, "ts": 1514656750115.663,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750117.225, "dur": 84.473,
    "args": {
      "External id": 2483380,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750118.680, "dur": 20.138,
    "args": {
      "External id": 2483381,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750119.549, "dur": 2.079,
    "args": {
      "External id": 2483382,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750125.074, "dur": 13.484,
    "args": {
      "External id": 2483383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750125.560, "dur": 12.674,
    "args": {
      "External id": 2483384,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750139.747, "dur": 3.688,
    "args": {
      "External id": 2483385,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750142.732, "dur": 0.524,
    "args": {
      "External id": 2483386,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750144.067, "dur": 56.813,
    "args": {
      "External id": 2483387,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750208.804, "dur": 89.550,
    "args": {
      "External id": 2483388,"Record function id": 0, "Sequence number": 24740626, "Fwd thread id": 1, "Ev Idx": 187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750209.928, "dur": 72.353,
    "args": {
      "External id": 2483389,"Sequence number": 24740626, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 188
    }
  },
  {
    "ph": "f", "id": 22, "pid": 1336760, "tid": 1381209, "ts": 1514656750209.928,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750212.965, "dur": 69.023,
    "args": {
      "External id": 2483390,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750213.775, "dur": 16.562,
    "args": {
      "External id": 2483391,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750214.580, "dur": 1.582,
    "args": {
      "External id": 2483392,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750217.067, "dur": 12.983,
    "args": {
      "External id": 2483393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750219.870, "dur": 9.872,
    "args": {
      "External id": 2483394,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750231.084, "dur": 3.663,
    "args": {
      "External id": 2483395,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750234.199, "dur": 0.406,
    "args": {
      "External id": 2483396,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750235.425, "dur": 45.916,
    "args": {
      "External id": 2483397,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750285.188, "dur": 12.025,
    "args": {
      "External id": 2483398,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750302.512, "dur": 30.845,
    "args": {
      "External id": 2483399,"Record function id": 0, "Sequence number": 24740625, "Fwd thread id": 1, "Ev Idx": 198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750303.853, "dur": 1.042,
    "args": {
      "External id": 2483400,"Sequence number": 24740625, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 199
    }
  },
  {
    "ph": "f", "id": 23, "pid": 1336760, "tid": 1381209, "ts": 1514656750303.853,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750307.066, "dur": 21.417,
    "args": {
      "External id": 2483401,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750308.674, "dur": 19.452,
    "args": {
      "External id": 2483402,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750313.185, "dur": 0.630,
    "args": {
      "External id": 2483403,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750337.743, "dur": 3095.505,
    "args": {
      "External id": 2483404,"Record function id": 0, "Sequence number": 24740624, "Fwd thread id": 1, "Ev Idx": 203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750349.832, "dur": 3054.609,
    "args": {
      "External id": 2483405,"Sequence number": 24740624, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 204
    }
  },
  {
    "ph": "f", "id": 24, "pid": 1336760, "tid": 1381209, "ts": 1514656750349.832,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750378.213, "dur": 2.464,
    "args": {
      "External id": 2483406,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750382.419, "dur": 2941.295,
    "args": {
      "External id": 2483407,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750383.726, "dur": 2939.660,
    "args": {
      "External id": 2483408,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750387.588, "dur": 3.330,
    "args": {
      "External id": 2483409,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656750391.503, "dur": 2931.150,
    "args": {
      "External id": 2483410,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753326.450, "dur": 0.386,
    "args": {
      "External id": 2483411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753328.099, "dur": 2.232,
    "args": {
      "External id": 2483412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753329.374, "dur": 0.837,
    "args": {
      "External id": 2483413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753334.051, "dur": 21.429,
    "args": {
      "External id": 2483414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753359.610, "dur": 38.505,
    "args": {
      "External id": 2483415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753360.975, "dur": 36.966,
    "args": {
      "External id": 2483416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753361.940, "dur": 35.663,
    "args": {
      "External id": 2483417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753414.615, "dur": 13.137,
    "args": {
      "External id": 2483418,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753447.384, "dur": 14.433,
    "args": {
      "External id": 2483419,"Record function id": 0, "Ev Idx": 218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753450.314, "dur": 9.198,
    "args": {
      "External id": 2483420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753453.767, "dur": 4.430,
    "args": {
      "External id": 2483421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753454.527, "dur": 3.538,
    "args": {
      "External id": 2483422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753465.763, "dur": 12.407,
    "args": {
      "External id": 2483423,"Record function id": 0, "Sequence number": 24740623, "Fwd thread id": 1, "Ev Idx": 222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753466.885, "dur": 8.279,
    "args": {
      "External id": 2483424,"Sequence number": 24740623, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 223
    }
  },
  {
    "ph": "f", "id": 25, "pid": 1336760, "tid": 1381209, "ts": 1514656753466.885,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753469.207, "dur": 5.752,
    "args": {
      "External id": 2483425,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753472.510, "dur": 2.298,
    "args": {
      "External id": 2483426,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753481.322, "dur": 86.742,
    "args": {
      "External id": 2483427,"Record function id": 0, "Sequence number": 24740622, "Fwd thread id": 1, "Ev Idx": 226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753482.367, "dur": 79.686,
    "args": {
      "External id": 2483428,"Sequence number": 24740622, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 227
    }
  },
  {
    "ph": "f", "id": 26, "pid": 1336760, "tid": 1381209, "ts": 1514656753482.367,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753484.377, "dur": 77.267,
    "args": {
      "External id": 2483429,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753488.756, "dur": 22.967,
    "args": {
      "External id": 2483430,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753490.337, "dur": 2.121,
    "args": {
      "External id": 2483431,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753493.139, "dur": 18.276,
    "args": {
      "External id": 2483432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753494.685, "dur": 16.235,
    "args": {
      "External id": 2483433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753513.193, "dur": 5.293,
    "args": {
      "External id": 2483434,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753517.192, "dur": 1.017,
    "args": {
      "External id": 2483435,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753519.347, "dur": 41.193,
    "args": {
      "External id": 2483436,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753576.639, "dur": 57.974,
    "args": {
      "External id": 2483437,"Record function id": 0, "Sequence number": 24740621, "Fwd thread id": 1, "Ev Idx": 236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753577.974, "dur": 53.829,
    "args": {
      "External id": 2483438,"Sequence number": 24740621, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 237
    }
  },
  {
    "ph": "f", "id": 27, "pid": 1336760, "tid": 1381209, "ts": 1514656753577.974,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753580.226, "dur": 51.293,
    "args": {
      "External id": 2483439,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753581.885, "dur": 22.841,
    "args": {
      "External id": 2483440,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753588.747, "dur": 3.375,
    "args": {
      "External id": 2483441,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753592.972, "dur": 11.502,
    "args": {
      "External id": 2483442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753594.072, "dur": 10.042,
    "args": {
      "External id": 2483443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753605.969, "dur": 4.553,
    "args": {
      "External id": 2483444,"Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753608.991, "dur": 1.018,
    "args": {
      "External id": 2483445,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753611.225, "dur": 19.768,
    "args": {
      "External id": 2483446,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753638.477, "dur": 89.437,
    "args": {
      "External id": 2483447,"Record function id": 0, "Sequence number": 24740620, "Fwd thread id": 1, "Ev Idx": 246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753639.502, "dur": 85.378,
    "args": {
      "External id": 2483448,"Sequence number": 24740620, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 247
    }
  },
  {
    "ph": "f", "id": 28, "pid": 1336760, "tid": 1381209, "ts": 1514656753639.502,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753641.407, "dur": 83.159,
    "args": {
      "External id": 2483449,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753644.657, "dur": 16.652,
    "args": {
      "External id": 2483450,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753645.716, "dur": 2.160,
    "args": {
      "External id": 2483451,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753650.923, "dur": 10.136,
    "args": {
      "External id": 2483452,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753651.553, "dur": 9.155,
    "args": {
      "External id": 2483453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753662.415, "dur": 4.444,
    "args": {
      "External id": 2483454,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753663.571, "dur": 3.071,
    "args": {
      "External id": 2483455,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753667.408, "dur": 56.348,
    "args": {
      "External id": 2483456,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753731.635, "dur": 94.832,
    "args": {
      "External id": 2483457,"Record function id": 0, "Sequence number": 24740619, "Fwd thread id": 1, "Ev Idx": 256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753732.791, "dur": 76.484,
    "args": {
      "External id": 2483458,"Sequence number": 24740619, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 257
    }
  },
  {
    "ph": "f", "id": 29, "pid": 1336760, "tid": 1381209, "ts": 1514656753732.791,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753734.344, "dur": 74.689,
    "args": {
      "External id": 2483459,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753735.312, "dur": 25.445,
    "args": {
      "External id": 2483460,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753739.235, "dur": 1.613,
    "args": {
      "External id": 2483461,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753744.930, "dur": 15.576,
    "args": {
      "External id": 2483462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753749.330, "dur": 10.834,
    "args": {
      "External id": 2483463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753761.637, "dur": 1.682,
    "args": {
      "External id": 2483464,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753762.706, "dur": 0.477,
    "args": {
      "External id": 2483465,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753763.914, "dur": 44.571,
    "args": {
      "External id": 2483466,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753813.379, "dur": 11.146,
    "args": {
      "External id": 2483467,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753831.908, "dur": 427.998,
    "args": {
      "External id": 2483468,"Record function id": 0, "Sequence number": 24740618, "Fwd thread id": 1, "Ev Idx": 267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656753833.285, "dur": 416.437,
    "args": {
      "External id": 2483469,"Sequence number": 24740618, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 268
    }
  },
  {
    "ph": "f", "id": 30, "pid": 1336760, "tid": 1381209, "ts": 1514656753833.285,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754069.983, "dur": 49.529,
    "args": {
      "External id": 2483470,"kernel_hash": "ceuhgo46kaqbnwlditj3h4z52obosunxjs35ou76xc47fiiswkar", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "262144", "2048", "1", "1986", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/eu/ceuhgo46kaqbnwlditj3h4z52obosunxjs35ou76xc47fiiswkar.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [2048], [262144, 2048], [262144, 2048], [132, 2048], [262144], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754159.425, "dur": 28.338,
    "args": {
      "External id": 2483471,"kernel_hash": "c4akubwlpa24tvlbvx7jydmsxo5wvd2yhzmkuenzmf65zqjkqtyl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/4a/c4akubwlpa24tvlbvx7jydmsxo5wvd2yhzmkuenzmf65zqjkqtyl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754205.040, "dur": 17.240,
    "args": {
      "External id": 2483472,"kernel_hash": "cdbhfghffo4a54xshvxcoylttdt6djirisy6yibfmduah5ybzscb", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/db/cdbhfghffo4a54xshvxcoylttdt6djirisy6yibfmduah5ybzscb.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754271.412, "dur": 13.848,
    "args": {
      "External id": 2483473,"Record function id": 0, "Ev Idx": 272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754273.712, "dur": 10.708,
    "args": {
      "External id": 2483474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754277.056, "dur": 6.433,
    "args": {
      "External id": 2483475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754280.080, "dur": 3.284,
    "args": {
      "External id": 2483476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: StackBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754291.535, "dur": 40.701,
    "args": {
      "External id": 2483477,"Record function id": 0, "Sequence number": 24740617, "Fwd thread id": 1, "Ev Idx": 276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "StackBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754292.766, "dur": 28.341,
    "args": {
      "External id": 2483478,"Sequence number": 24740617, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 277
    }
  },
  {
    "ph": "f", "id": 31, "pid": 1336760, "tid": 1381209, "ts": 1514656754292.766,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754295.104, "dur": 8.971,
    "args": {
      "External id": 2483479,"Record function id": 0, "Concrete Inputs": ["", "-2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754300.882, "dur": 1.243,
    "args": {
      "External id": 2483480,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754304.715, "dur": 7.115,
    "args": {
      "External id": 2483481,"Record function id": 0, "Concrete Inputs": ["", "-2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754310.211, "dur": 0.722,
    "args": {
      "External id": 2483482,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754312.717, "dur": 2.553,
    "args": {
      "External id": 2483483,"Record function id": 0, "Concrete Inputs": ["", "-2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754313.763, "dur": 0.655,
    "args": {
      "External id": 2483484,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754315.737, "dur": 4.727,
    "args": {
      "External id": 2483485,"Record function id": 0, "Concrete Inputs": ["", "-2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754318.793, "dur": 0.722,
    "args": {
      "External id": 2483486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754336.122, "dur": 7.911,
    "args": {
      "External id": 2483487,"Record function id": 0, "Sequence number": 24740616, "Fwd thread id": 1, "Ev Idx": 286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754337.197, "dur": 1.130,
    "args": {
      "External id": 2483488,"Sequence number": 24740616, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 287
    }
  },
  {
    "ph": "f", "id": 32, "pid": 1336760, "tid": 1381209, "ts": 1514656754337.197,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754348.614, "dur": 596.726,
    "args": {
      "External id": 2483489,"Record function id": 0, "Sequence number": 24740615, "Fwd thread id": 1, "Ev Idx": 288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754350.025, "dur": 580.692,
    "args": {
      "External id": 2483490,"Sequence number": 24740615, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 289
    }
  },
  {
    "ph": "f", "id": 33, "pid": 1336760, "tid": 1381209, "ts": 1514656754350.025,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754400.406, "dur": 14.129,
    "args": {
      "External id": 2483491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754409.287, "dur": 4.912,
    "args": {
      "External id": 2483492,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754421.992, "dur": 7.157,
    "args": {
      "External id": 2483493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754425.483, "dur": 2.597,
    "args": {
      "External id": 2483494,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754427.090, "dur": 0.766,
    "args": {
      "External id": 2483495,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754437.109, "dur": 164.854,
    "args": {
      "External id": 2483496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754438.564, "dur": 7.315,
    "args": {
      "External id": 2483497,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754439.223, "dur": 5.825,
    "args": {
      "External id": 2483498,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754442.245, "dur": 2.692,
    "args": {
      "External id": 2483499,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754448.403, "dur": 152.557,
    "args": {
      "External id": 2483500,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754451.484, "dur": 147.883,
    "args": {
      "External id": 2483501,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754610.330, "dur": 7.509,
    "args": {
      "External id": 2483502,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754614.832, "dur": 2.862,
    "args": {
      "External id": 2483503,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754661.225, "dur": 5.168,
    "args": {
      "External id": 2483504,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754667.514, "dur": 4.819,
    "args": {
      "External id": 2483505,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754675.499, "dur": 1.732,
    "args": {
      "External id": 2483506,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754715.413, "dur": 2.522,
    "args": {
      "External id": 2483507,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754716.146, "dur": 1.653,
    "args": {
      "External id": 2483508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754748.710, "dur": 157.706,
    "args": {
      "External id": 2483509,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754757.000, "dur": 10.585,
    "args": {
      "External id": 2483510,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754763.686, "dur": 1.189,
    "args": {
      "External id": 2483511,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754769.756, "dur": 9.555,
    "args": {
      "External id": 2483512,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754775.826, "dur": 2.628,
    "args": {
      "External id": 2483513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754781.518, "dur": 3.221,
    "args": {
      "External id": 2483514,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754783.896, "dur": 0.479,
    "args": {
      "External id": 2483515,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754785.407, "dur": 3.960,
    "args": {
      "External id": 2483516,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754788.190, "dur": 0.442,
    "args": {
      "External id": 2483517,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754794.576, "dur": 3.833,
    "args": {
      "External id": 2483518,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754797.538, "dur": 0.562,
    "args": {
      "External id": 2483519,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754801.170, "dur": 8.427,
    "args": {
      "External id": 2483520,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754807.069, "dur": 2.341,
    "args": {
      "External id": 2483521,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754810.296, "dur": 3.565,
    "args": {
      "External id": 2483522,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754812.895, "dur": 0.649,
    "args": {
      "External id": 2483523,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754814.315, "dur": 2.156,
    "args": {
      "External id": 2483524,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754815.103, "dur": 1.273,
    "args": {
      "External id": 2483525,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754818.593, "dur": 73.996,
    "args": {
      "External id": 2483526,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754894.138, "dur": 3.543,
    "args": {
      "External id": 2483527,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754900.315, "dur": 2.146,
    "args": {
      "External id": 2483528,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754901.345, "dur": 0.485,
    "args": {
      "External id": 2483529,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754904.358, "dur": 0.921,
    "args": {
      "External id": 2483530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754958.429, "dur": 10.080,
    "args": {
      "External id": 2483531,"Record function id": 0, "Ev Idx": 330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754960.680, "dur": 7.085,
    "args": {
      "External id": 2483532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754963.698, "dur": 3.269,
    "args": {
      "External id": 2483533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754964.609, "dur": 2.244,
    "args": {
      "External id": 2483534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754972.705, "dur": 46.067,
    "args": {
      "External id": 2483535,"Record function id": 0, "Sequence number": 24740614, "Fwd thread id": 1, "Ev Idx": 334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754973.673, "dur": 5.478,
    "args": {
      "External id": 2483536,"Sequence number": 24740614, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 335
    }
  },
  {
    "ph": "f", "id": 34, "pid": 1336760, "tid": 1381209, "ts": 1514656754973.673,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754977.202, "dur": 1.761,
    "args": {
      "External id": 2483537,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656754977.807, "dur": 1.039,
    "args": {
      "External id": 2483538,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755025.597, "dur": 186.376,
    "args": {
      "External id": 2483539,"Record function id": 0, "Sequence number": 24740613, "Fwd thread id": 1, "Ev Idx": 338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755026.669, "dur": 175.348,
    "args": {
      "External id": 2483540,"Sequence number": 24740613, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 339
    }
  },
  {
    "ph": "f", "id": 35, "pid": 1336760, "tid": 1381209, "ts": 1514656755026.669,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755034.908, "dur": 5.298,
    "args": {
      "External id": 2483541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755036.717, "dur": 2.664,
    "args": {
      "External id": 2483542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755038.233, "dur": 0.911,
    "args": {
      "External id": 2483543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755041.715, "dur": 65.878,
    "args": {
      "External id": 2483544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755109.159, "dur": 6.547,
    "args": {
      "External id": 2483545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755109.856, "dur": 5.220,
    "args": {
      "External id": 2483546,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755113.504, "dur": 1.398,
    "args": {
      "External id": 2483547,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755118.075, "dur": 5.691,
    "args": {
      "External id": 2483548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755119.155, "dur": 4.062,
    "args": {
      "External id": 2483549,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755122.393, "dur": 0.760,
    "args": {
      "External id": 2483550,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755124.402, "dur": 76.657,
    "args": {
      "External id": 2483551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755217.189, "dur": 7.886,
    "args": {
      "External id": 2483552,"Record function id": 0, "Sequence number": 24740612, "Fwd thread id": 1, "Ev Idx": 351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755218.451, "dur": 4.456,
    "args": {
      "External id": 2483553,"Sequence number": 24740612, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 352
    }
  },
  {
    "ph": "f", "id": 36, "pid": 1336760, "tid": 1381209, "ts": 1514656755218.451,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755220.247, "dur": 2.510,
    "args": {
      "External id": 2483554,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755221.204, "dur": 1.418,
    "args": {
      "External id": 2483555,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755229.172, "dur": 12.328,
    "args": {
      "External id": 2483556,"Record function id": 0, "Sequence number": 24740611, "Fwd thread id": 1, "Ev Idx": 355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755232.078, "dur": 6.165,
    "args": {
      "External id": 2483557,"Sequence number": 24740611, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 356
    }
  },
  {
    "ph": "f", "id": 37, "pid": 1336760, "tid": 1381209, "ts": 1514656755232.078,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755233.076, "dur": 4.949,
    "args": {
      "External id": 2483558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755233.974, "dur": 3.573,
    "args": {
      "External id": 2483559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755236.905, "dur": 0.511,
    "args": {
      "External id": 2483560,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755245.872, "dur": 5.521,
    "args": {
      "External id": 2483561,"Record function id": 0, "Ev Idx": 360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755247.163, "dur": 3.704,
    "args": {
      "External id": 2483562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755248.331, "dur": 2.160,
    "args": {
      "External id": 2483563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755248.974, "dur": 1.402,
    "args": {
      "External id": 2483564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755254.655, "dur": 8.618,
    "args": {
      "External id": 2483565,"Record function id": 0, "Sequence number": 24740610, "Fwd thread id": 1, "Ev Idx": 364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755255.496, "dur": 4.835,
    "args": {
      "External id": 2483566,"Sequence number": 24740610, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 365
    }
  },
  {
    "ph": "f", "id": 38, "pid": 1336760, "tid": 1381209, "ts": 1514656755255.496,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755256.577, "dur": 3.626,
    "args": {
      "External id": 2483567,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755259.422, "dur": 0.669,
    "args": {
      "External id": 2483568,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755266.337, "dur": 122.826,
    "args": {
      "External id": 2483569,"Record function id": 0, "Sequence number": 24740609, "Fwd thread id": 1, "Ev Idx": 368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755267.321, "dur": 112.025,
    "args": {
      "External id": 2483570,"Sequence number": 24740609, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 369
    }
  },
  {
    "ph": "f", "id": 39, "pid": 1336760, "tid": 1381209, "ts": 1514656755267.321,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755270.671, "dur": 2.571,
    "args": {
      "External id": 2483571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755271.319, "dur": 1.490,
    "args": {
      "External id": 2483572,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755272.245, "dur": 0.455,
    "args": {
      "External id": 2483573,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755276.463, "dur": 37.201,
    "args": {
      "External id": 2483574,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755314.696, "dur": 5.529,
    "args": {
      "External id": 2483575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755315.584, "dur": 4.147,
    "args": {
      "External id": 2483576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755318.675, "dur": 0.950,
    "args": {
      "External id": 2483577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755321.674, "dur": 8.240,
    "args": {
      "External id": 2483578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755322.882, "dur": 6.278,
    "args": {
      "External id": 2483579,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755326.378, "dur": 2.701,
    "args": {
      "External id": 2483580,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755330.677, "dur": 47.814,
    "args": {
      "External id": 2483581,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755396.199, "dur": 38.128,
    "args": {
      "External id": 2483582,"Record function id": 0, "Sequence number": 24740608, "Fwd thread id": 1, "Ev Idx": 381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755397.489, "dur": 5.794,
    "args": {
      "External id": 2483583,"Sequence number": 24740608, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 382
    }
  },
  {
    "ph": "f", "id": 40, "pid": 1336760, "tid": 1381209, "ts": 1514656755397.489,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755398.824, "dur": 4.315,
    "args": {
      "External id": 2483584,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755401.864, "dur": 1.133,
    "args": {
      "External id": 2483585,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755407.676, "dur": 23.717,
    "args": {
      "External id": 2483586,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755438.626, "dur": 8.971,
    "args": {
      "External id": 2483587,"Record function id": 0, "Sequence number": 24740607, "Fwd thread id": 1, "Ev Idx": 386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755439.885, "dur": 5.972,
    "args": {
      "External id": 2483588,"Sequence number": 24740607, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 387
    }
  },
  {
    "ph": "f", "id": 41, "pid": 1336760, "tid": 1381209, "ts": 1514656755439.885,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755440.904, "dur": 4.772,
    "args": {
      "External id": 2483589,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755443.252, "dur": 1.938,
    "args": {
      "External id": 2483590,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755444.482, "dur": 0.606,
    "args": {
      "External id": 2483591,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755451.435, "dur": 5.566,
    "args": {
      "External id": 2483592,"Record function id": 0, "Ev Idx": 391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755452.837, "dur": 3.697,
    "args": {
      "External id": 2483593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755454.141, "dur": 1.985,
    "args": {
      "External id": 2483594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755454.834, "dur": 1.200,
    "args": {
      "External id": 2483595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755461.511, "dur": 517.666,
    "args": {
      "External id": 2483596,"Record function id": 0, "Sequence number": 24740606, "Fwd thread id": 1, "Ev Idx": 395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755462.846, "dur": 497.299,
    "args": {
      "External id": 2483597,"Sequence number": 24740606, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 396
    }
  },
  {
    "ph": "f", "id": 42, "pid": 1336760, "tid": 1381209, "ts": 1514656755462.846,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755492.889, "dur": 48.176,
    "args": {
      "External id": 2483598,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755495.119, "dur": 45.729,
    "args": {
      "External id": 2483599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755498.632, "dur": 6.474,
    "args": {
      "External id": 2483600,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755501.256, "dur": 3.149,
    "args": {
      "External id": 2483601,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755506.609, "dur": 33.425,
    "args": {
      "External id": 2483602,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755556.589, "dur": 4.663,
    "args": {
      "External id": 2483603,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755559.409, "dur": 1.730,
    "args": {
      "External id": 2483604,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755565.515, "dur": 3.729,
    "args": {
      "External id": 2483605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755566.228, "dur": 2.931,
    "args": {
      "External id": 2483606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755586.788, "dur": 3.147,
    "args": {
      "External id": 2483607,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755606.705, "dur": 3.308,
    "args": {
      "External id": 2483608,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755841.291, "dur": 2.674,
    "args": {
      "External id": 2483609,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755848.948, "dur": 38.180,
    "args": {
      "External id": 2483610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755861.413, "dur": 1.210,
    "args": {
      "External id": 2483611,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755894.119, "dur": 29.878,
    "args": {
      "External id": 2483612,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755896.377, "dur": 27.379,
    "args": {
      "External id": 2483613,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755900.344, "dur": 4.065,
    "args": {
      "External id": 2483614,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755905.810, "dur": 17.471,
    "args": {
      "External id": 2483615,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755930.950, "dur": 3.376,
    "args": {
      "External id": 2483616,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755932.446, "dur": 1.762,
    "args": {
      "External id": 2483617,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755940.781, "dur": 3.811,
    "args": {
      "External id": 2483618,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755943.257, "dur": 1.233,
    "args": {
      "External id": 2483619,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755946.509, "dur": 4.021,
    "args": {
      "External id": 2483620,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656755947.324, "dur": 3.113,
    "args": {
      "External id": 2483621,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756035.155, "dur": 11.537,
    "args": {
      "External id": 2483622,"Record function id": 0, "Ev Idx": 421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756037.711, "dur": 8.053,
    "args": {
      "External id": 2483623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756040.266, "dur": 3.970,
    "args": {
      "External id": 2483624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756041.331, "dur": 2.688,
    "args": {
      "External id": 2483625,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756050.303, "dur": 11.870,
    "args": {
      "External id": 2483626,"Record function id": 0, "Sequence number": 24740605, "Fwd thread id": 1, "Ev Idx": 425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756051.578, "dur": 7.151,
    "args": {
      "External id": 2483627,"Sequence number": 24740605, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 426
    }
  },
  {
    "ph": "f", "id": 43, "pid": 1336760, "tid": 1381209, "ts": 1514656756051.578,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756055.659, "dur": 2.780,
    "args": {
      "External id": 2483628,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756056.546, "dur": 1.799,
    "args": {
      "External id": 2483629,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756068.367, "dur": 151.899,
    "args": {
      "External id": 2483630,"Record function id": 0, "Sequence number": 24740604, "Fwd thread id": 1, "Ev Idx": 429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756069.613, "dur": 143.832,
    "args": {
      "External id": 2483631,"Sequence number": 24740604, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 430
    }
  },
  {
    "ph": "f", "id": 44, "pid": 1336760, "tid": 1381209, "ts": 1514656756069.613,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756074.524, "dur": 5.122,
    "args": {
      "External id": 2483632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756076.326, "dur": 2.802,
    "args": {
      "External id": 2483633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756077.955, "dur": 0.962,
    "args": {
      "External id": 2483634,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756081.003, "dur": 69.038,
    "args": {
      "External id": 2483635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756151.184, "dur": 7.429,
    "args": {
      "External id": 2483636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756153.862, "dur": 4.133,
    "args": {
      "External id": 2483637,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756155.117, "dur": 2.704,
    "args": {
      "External id": 2483638,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756160.473, "dur": 5.129,
    "args": {
      "External id": 2483639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756161.249, "dur": 3.816,
    "args": {
      "External id": 2483640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756164.469, "dur": 0.506,
    "args": {
      "External id": 2483641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756166.262, "dur": 46.289,
    "args": {
      "External id": 2483642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756225.776, "dur": 11.421,
    "args": {
      "External id": 2483643,"Record function id": 0, "Sequence number": 24740603, "Fwd thread id": 1, "Ev Idx": 442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756226.942, "dur": 7.668,
    "args": {
      "External id": 2483644,"Sequence number": 24740603, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 443
    }
  },
  {
    "ph": "f", "id": 45, "pid": 1336760, "tid": 1381209, "ts": 1514656756226.942,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756230.703, "dur": 3.756,
    "args": {
      "External id": 2483645,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756231.355, "dur": 2.980,
    "args": {
      "External id": 2483646,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756240.663, "dur": 10.294,
    "args": {
      "External id": 2483647,"Record function id": 0, "Sequence number": 24740602, "Fwd thread id": 1, "Ev Idx": 446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756242.125, "dur": 6.437,
    "args": {
      "External id": 2483648,"Sequence number": 24740602, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 447
    }
  },
  {
    "ph": "f", "id": 46, "pid": 1336760, "tid": 1381209, "ts": 1514656756242.125,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756243.201, "dur": 5.174,
    "args": {
      "External id": 2483649,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756243.963, "dur": 3.932,
    "args": {
      "External id": 2483650,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756247.108, "dur": 0.700,
    "args": {
      "External id": 2483651,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756254.784, "dur": 7.193,
    "args": {
      "External id": 2483652,"Record function id": 0, "Ev Idx": 451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756256.023, "dur": 5.445,
    "args": {
      "External id": 2483653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756256.941, "dur": 4.211,
    "args": {
      "External id": 2483654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756259.715, "dur": 1.333,
    "args": {
      "External id": 2483655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756265.271, "dur": 8.075,
    "args": {
      "External id": 2483656,"Record function id": 0, "Sequence number": 24740601, "Fwd thread id": 1, "Ev Idx": 455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756266.043, "dur": 3.713,
    "args": {
      "External id": 2483657,"Sequence number": 24740601, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 456
    }
  },
  {
    "ph": "f", "id": 47, "pid": 1336760, "tid": 1381209, "ts": 1514656756266.043,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756267.416, "dur": 2.191,
    "args": {
      "External id": 2483658,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756268.261, "dur": 1.219,
    "args": {
      "External id": 2483659,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756278.091, "dur": 364.310,
    "args": {
      "External id": 2483660,"Record function id": 0, "Sequence number": 24740600, "Fwd thread id": 1, "Ev Idx": 459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756279.374, "dur": 338.910,
    "args": {
      "External id": 2483661,"Sequence number": 24740600, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 460
    }
  },
  {
    "ph": "f", "id": 48, "pid": 1336760, "tid": 1381209, "ts": 1514656756279.374,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756301.155, "dur": 11.350,
    "args": {
      "External id": 2483662,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756308.247, "dur": 3.820,
    "args": {
      "External id": 2483663,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756314.441, "dur": 3.764,
    "args": {
      "External id": 2483664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756315.767, "dur": 2.236,
    "args": {
      "External id": 2483665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756319.812, "dur": 6.504,
    "args": {
      "External id": 2483666,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756322.694, "dur": 3.424,
    "args": {
      "External id": 2483667,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756359.506, "dur": 228.779,
    "args": {
      "External id": 2483668,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756461.389, "dur": 3.729,
    "args": {
      "External id": 2483669,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756467.022, "dur": 3.963,
    "args": {
      "External id": 2483670,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756604.479, "dur": 4.316,
    "args": {
      "External id": 2483671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756611.888, "dur": 0.817,
    "args": {
      "External id": 2483672,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756614.903, "dur": 0.707,
    "args": {
      "External id": 2483673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756652.542, "dur": 261.783,
    "args": {
      "External id": 2483674,"Record function id": 0, "Sequence number": 24740599, "Fwd thread id": 1, "Ev Idx": 473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756654.402, "dur": 252.052,
    "args": {
      "External id": 2483675,"Sequence number": 24740599, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 474
    }
  },
  {
    "ph": "f", "id": 49, "pid": 1336760, "tid": 1381209, "ts": 1514656756654.402,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756678.707, "dur": 53.963,
    "args": {
      "External id": 2483676,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756685.648, "dur": 3.810,
    "args": {
      "External id": 2483677,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756690.960, "dur": 41.058,
    "args": {
      "External id": 2483678,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756743.822, "dur": 3.949,
    "args": {
      "External id": 2483679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756745.210, "dur": 2.261,
    "args": {
      "External id": 2483680,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756921.009, "dur": 219.002,
    "args": {
      "External id": 2483681,"Record function id": 0, "Sequence number": 24740598, "Fwd thread id": 1, "Ev Idx": 480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756923.171, "dur": 209.360,
    "args": {
      "External id": 2483682,"Sequence number": 24740598, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 481
    }
  },
  {
    "ph": "f", "id": 50, "pid": 1336760, "tid": 1381209, "ts": 1514656756923.171,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756936.853, "dur": 32.946,
    "args": {
      "External id": 2483683,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756941.414, "dur": 2.752,
    "args": {
      "External id": 2483684,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756945.190, "dur": 24.128,
    "args": {
      "External id": 2483685,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756976.625, "dur": 53.831,
    "args": {
      "External id": 2483686,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656756977.973, "dur": 51.767,
    "args": {
      "External id": 2483687,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757151.222, "dur": 17.492,
    "args": {
      "External id": 2483688,"Record function id": 0, "Sequence number": 24740597, "Fwd thread id": 1, "Ev Idx": 487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757152.801, "dur": 12.647,
    "args": {
      "External id": 2483689,"Sequence number": 24740597, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 488
    }
  },
  {
    "ph": "f", "id": 51, "pid": 1336760, "tid": 1381209, "ts": 1514656757152.801,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757155.722, "dur": 9.461,
    "args": {
      "External id": 2483690,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757157.290, "dur": 7.680,
    "args": {
      "External id": 2483691,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757172.376, "dur": 9.656,
    "args": {
      "External id": 2483692,"Record function id": 0, "Sequence number": 24740596, "Fwd thread id": 1, "Ev Idx": 491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757173.483, "dur": 5.555,
    "args": {
      "External id": 2483693,"Sequence number": 24740596, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 492
    }
  },
  {
    "ph": "f", "id": 52, "pid": 1336760, "tid": 1381209, "ts": 1514656757173.483,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757177.282, "dur": 1.612,
    "args": {
      "External id": 2483694,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757177.888, "dur": 0.865,
    "args": {
      "External id": 2483695,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757185.250, "dur": 8.963,
    "args": {
      "External id": 2483696,"Record function id": 0, "Sequence number": 24740595, "Fwd thread id": 1, "Ev Idx": 495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757186.453, "dur": 5.184,
    "args": {
      "External id": 2483697,"Sequence number": 24740595, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 496
    }
  },
  {
    "ph": "f", "id": 53, "pid": 1336760, "tid": 1381209, "ts": 1514656757186.453,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757189.927, "dur": 1.569,
    "args": {
      "External id": 2483698,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757190.436, "dur": 0.946,
    "args": {
      "External id": 2483699,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757199.361, "dur": 6.425,
    "args": {
      "External id": 2483700,"Record function id": 0, "Sequence number": 24740594, "Fwd thread id": 1, "Ev Idx": 499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757200.511, "dur": 3.266,
    "args": {
      "External id": 2483701,"Sequence number": 24740594, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 500
    }
  },
  {
    "ph": "f", "id": 54, "pid": 1336760, "tid": 1381209, "ts": 1514656757200.511,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757201.806, "dur": 1.839,
    "args": {
      "External id": 2483702,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757202.608, "dur": 0.941,
    "args": {
      "External id": 2483703,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757209.375, "dur": 184.783,
    "args": {
      "External id": 2483704,"Record function id": 0, "Sequence number": 24740593, "Fwd thread id": 1, "Ev Idx": 503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757210.651, "dur": 174.675,
    "args": {
      "External id": 2483705,"Sequence number": 24740593, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 504
    }
  },
  {
    "ph": "f", "id": 55, "pid": 1336760, "tid": 1381209, "ts": 1514656757210.651,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757216.119, "dur": 9.375,
    "args": {
      "External id": 2483706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757220.457, "dur": 4.424,
    "args": {
      "External id": 2483707,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757222.940, "dur": 1.645,
    "args": {
      "External id": 2483708,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757227.128, "dur": 68.669,
    "args": {
      "External id": 2483709,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757297.358, "dur": 6.885,
    "args": {
      "External id": 2483710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757298.384, "dur": 5.254,
    "args": {
      "External id": 2483711,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757302.112, "dur": 1.366,
    "args": {
      "External id": 2483712,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757331.979, "dur": 5.621,
    "args": {
      "External id": 2483713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757335.755, "dur": 1.111,
    "args": {
      "External id": 2483714,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757336.296, "dur": 0.482,
    "args": {
      "External id": 2483715,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757338.335, "dur": 45.867,
    "args": {
      "External id": 2483716,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757399.512, "dur": 6.720,
    "args": {
      "External id": 2483717,"Record function id": 0, "Sequence number": 24740592, "Fwd thread id": 1, "Ev Idx": 516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757400.572, "dur": 3.264,
    "args": {
      "External id": 2483718,"Sequence number": 24740592, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 517
    }
  },
  {
    "ph": "f", "id": 56, "pid": 1336760, "tid": 1381209, "ts": 1514656757400.572,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757401.932, "dur": 1.770,
    "args": {
      "External id": 2483719,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757402.500, "dur": 1.109,
    "args": {
      "External id": 2483720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757409.556, "dur": 10.976,
    "args": {
      "External id": 2483721,"Record function id": 0, "Sequence number": 24740591, "Fwd thread id": 1, "Ev Idx": 520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757412.763, "dur": 5.201,
    "args": {
      "External id": 2483722,"Sequence number": 24740591, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 521
    }
  },
  {
    "ph": "f", "id": 57, "pid": 1336760, "tid": 1381209, "ts": 1514656757412.763,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757413.346, "dur": 4.443,
    "args": {
      "External id": 2483723,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757413.942, "dur": 3.397,
    "args": {
      "External id": 2483724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757416.826, "dur": 0.410,
    "args": {
      "External id": 2483725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757426.473, "dur": 8.529,
    "args": {
      "External id": 2483726,"Record function id": 0, "Ev Idx": 525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757427.747, "dur": 6.463,
    "args": {
      "External id": 2483727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757430.199, "dur": 3.612,
    "args": {
      "External id": 2483728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757431.124, "dur": 2.556,
    "args": {
      "External id": 2483729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757440.291, "dur": 7.748,
    "args": {
      "External id": 2483730,"Record function id": 0, "Sequence number": 24740590, "Fwd thread id": 1, "Ev Idx": 529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757441.123, "dur": 4.480,
    "args": {
      "External id": 2483731,"Sequence number": 24740590, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 530
    }
  },
  {
    "ph": "f", "id": 58, "pid": 1336760, "tid": 1381209, "ts": 1514656757441.123,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757442.277, "dur": 3.175,
    "args": {
      "External id": 2483732,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757444.641, "dur": 0.718,
    "args": {
      "External id": 2483733,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757451.454, "dur": 108.312,
    "args": {
      "External id": 2483734,"Record function id": 0, "Sequence number": 24740589, "Fwd thread id": 1, "Ev Idx": 533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757452.261, "dur": 99.042,
    "args": {
      "External id": 2483735,"Sequence number": 24740589, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 534
    }
  },
  {
    "ph": "f", "id": 59, "pid": 1336760, "tid": 1381209, "ts": 1514656757452.261,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757454.844, "dur": 4.300,
    "args": {
      "External id": 2483736,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757455.347, "dur": 3.339,
    "args": {
      "External id": 2483737,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757458.217, "dur": 0.367,
    "args": {
      "External id": 2483738,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757461.717, "dur": 27.638,
    "args": {
      "External id": 2483739,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757490.480, "dur": 2.810,
    "args": {
      "External id": 2483740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757490.996, "dur": 1.721,
    "args": {
      "External id": 2483741,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757491.951, "dur": 0.663,
    "args": {
      "External id": 2483742,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757494.374, "dur": 5.852,
    "args": {
      "External id": 2483743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757495.061, "dur": 4.658,
    "args": {
      "External id": 2483744,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757497.720, "dur": 1.887,
    "args": {
      "External id": 2483745,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757500.636, "dur": 49.315,
    "args": {
      "External id": 2483746,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757566.518, "dur": 36.003,
    "args": {
      "External id": 2483747,"Record function id": 0, "Sequence number": 24740588, "Fwd thread id": 1, "Ev Idx": 546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757567.455, "dur": 5.913,
    "args": {
      "External id": 2483748,"Sequence number": 24740588, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 547
    }
  },
  {
    "ph": "f", "id": 60, "pid": 1336760, "tid": 1381209, "ts": 1514656757567.455,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757570.922, "dur": 2.281,
    "args": {
      "External id": 2483749,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757571.430, "dur": 1.680,
    "args": {
      "External id": 2483750,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757576.653, "dur": 23.369,
    "args": {
      "External id": 2483751,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757606.461, "dur": 10.325,
    "args": {
      "External id": 2483752,"Record function id": 0, "Sequence number": 24740587, "Fwd thread id": 1, "Ev Idx": 551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757607.507, "dur": 6.001,
    "args": {
      "External id": 2483753,"Sequence number": 24740587, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 552
    }
  },
  {
    "ph": "f", "id": 61, "pid": 1336760, "tid": 1381209, "ts": 1514656757607.507,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757608.655, "dur": 4.653,
    "args": {
      "External id": 2483754,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757609.306, "dur": 3.430,
    "args": {
      "External id": 2483755,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757612.096, "dur": 0.554,
    "args": {
      "External id": 2483756,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757620.943, "dur": 7.680,
    "args": {
      "External id": 2483757,"Record function id": 0, "Ev Idx": 556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757624.323, "dur": 3.711,
    "args": {
      "External id": 2483758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757625.543, "dur": 2.038,
    "args": {
      "External id": 2483759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757626.041, "dur": 1.442,
    "args": {
      "External id": 2483760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757631.881, "dur": 6.837,
    "args": {
      "External id": 2483761,"Record function id": 0, "Sequence number": 24740586, "Fwd thread id": 1, "Ev Idx": 560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757632.843, "dur": 2.643,
    "args": {
      "External id": 2483762,"Sequence number": 24740586, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 561
    }
  },
  {
    "ph": "f", "id": 62, "pid": 1336760, "tid": 1381209, "ts": 1514656757632.843,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757633.986, "dur": 1.340,
    "args": {
      "External id": 2483763,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757634.370, "dur": 0.864,
    "args": {
      "External id": 2483764,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757641.757, "dur": 95.880,
    "args": {
      "External id": 2483765,"Record function id": 0, "Sequence number": 24740585, "Fwd thread id": 1, "Ev Idx": 564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757645.043, "dur": 83.531,
    "args": {
      "External id": 2483766,"Sequence number": 24740585, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 565
    }
  },
  {
    "ph": "f", "id": 63, "pid": 1336760, "tid": 1381209, "ts": 1514656757645.043,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757648.065, "dur": 2.114,
    "args": {
      "External id": 2483767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757648.513, "dur": 1.256,
    "args": {
      "External id": 2483768,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757649.133, "dur": 0.534,
    "args": {
      "External id": 2483769,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757650.940, "dur": 31.823,
    "args": {
      "External id": 2483770,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757683.811, "dur": 6.393,
    "args": {
      "External id": 2483771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757684.490, "dur": 5.115,
    "args": {
      "External id": 2483772,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757689.018, "dur": 0.465,
    "args": {
      "External id": 2483773,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757691.487, "dur": 2.796,
    "args": {
      "External id": 2483774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757692.414, "dur": 1.238,
    "args": {
      "External id": 2483775,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757693.158, "dur": 0.425,
    "args": {
      "External id": 2483776,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757694.794, "dur": 33.128,
    "args": {
      "External id": 2483777,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757742.047, "dur": 24.963,
    "args": {
      "External id": 2483778,"Record function id": 0, "Sequence number": 24740584, "Fwd thread id": 1, "Ev Idx": 577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757742.880, "dur": 5.277,
    "args": {
      "External id": 2483779,"Sequence number": 24740584, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 578
    }
  },
  {
    "ph": "f", "id": 64, "pid": 1336760, "tid": 1381209, "ts": 1514656757742.880,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757746.295, "dur": 1.720,
    "args": {
      "External id": 2483780,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757746.895, "dur": 0.982,
    "args": {
      "External id": 2483781,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757750.564, "dur": 14.353,
    "args": {
      "External id": 2483782,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757770.635, "dur": 11.283,
    "args": {
      "External id": 2483783,"Record function id": 0, "Sequence number": 24740583, "Fwd thread id": 1, "Ev Idx": 582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757771.694, "dur": 8.156,
    "args": {
      "External id": 2483784,"Sequence number": 24740583, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 583
    }
  },
  {
    "ph": "f", "id": 65, "pid": 1336760, "tid": 1381209, "ts": 1514656757771.694,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757772.610, "dur": 7.030,
    "args": {
      "External id": 2483785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757775.474, "dur": 3.632,
    "args": {
      "External id": 2483786,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757778.398, "dur": 0.554,
    "args": {
      "External id": 2483787,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757785.705, "dur": 6.840,
    "args": {
      "External id": 2483788,"Record function id": 0, "Ev Idx": 587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757786.712, "dur": 5.414,
    "args": {
      "External id": 2483789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757787.508, "dur": 4.278,
    "args": {
      "External id": 2483790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757790.436, "dur": 1.245,
    "args": {
      "External id": 2483791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757797.052, "dur": 417.865,
    "args": {
      "External id": 2483792,"Record function id": 0, "Sequence number": 24740582, "Fwd thread id": 1, "Ev Idx": 591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757798.086, "dur": 380.581,
    "args": {
      "External id": 2483793,"Sequence number": 24740582, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 592
    }
  },
  {
    "ph": "f", "id": 66, "pid": 1336760, "tid": 1381209, "ts": 1514656757798.086,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757832.224, "dur": 1.847,
    "args": {
      "External id": 2483794,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757832.930, "dur": 0.979,
    "args": {
      "External id": 2483795,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757850.205, "dur": 5.485,
    "args": {
      "External id": 2483796,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656757864.393, "dur": 2.649,
    "args": {
      "External id": 2483797,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758068.044, "dur": 3.074,
    "args": {
      "External id": 2483798,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758075.735, "dur": 40.242,
    "args": {
      "External id": 2483799,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758090.530, "dur": 1.098,
    "args": {
      "External id": 2483800,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758121.803, "dur": 30.385,
    "args": {
      "External id": 2483801,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758123.618, "dur": 28.371,
    "args": {
      "External id": 2483802,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758127.718, "dur": 5.717,
    "args": {
      "External id": 2483803,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758135.419, "dur": 16.104,
    "args": {
      "External id": 2483804,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758156.916, "dur": 2.711,
    "args": {
      "External id": 2483805,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758158.352, "dur": 1.121,
    "args": {
      "External id": 2483806,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758168.276, "dur": 3.719,
    "args": {
      "External id": 2483807,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758171.118, "dur": 0.791,
    "args": {
      "External id": 2483808,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758191.498, "dur": 16.693,
    "args": {
      "External id": 2483809,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758228.022, "dur": 8.697,
    "args": {
      "External id": 2483810,"Record function id": 0, "Ev Idx": 609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758229.919, "dur": 6.092,
    "args": {
      "External id": 2483811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758231.951, "dur": 3.077,
    "args": {
      "External id": 2483812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758233.019, "dur": 1.909,
    "args": {
      "External id": 2483813,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758242.984, "dur": 6.620,
    "args": {
      "External id": 2483814,"Record function id": 0, "Sequence number": 24740581, "Fwd thread id": 1, "Ev Idx": 613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758244.470, "dur": 1.436,
    "args": {
      "External id": 2483815,"Sequence number": 24740581, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 614
    }
  },
  {
    "ph": "f", "id": 67, "pid": 1336760, "tid": 1381209, "ts": 1514656758244.470,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758253.825, "dur": 445.570,
    "args": {
      "External id": 2483816,"Record function id": 0, "Sequence number": 24740580, "Fwd thread id": 1, "Ev Idx": 615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758255.068, "dur": 430.374,
    "args": {
      "External id": 2483817,"Sequence number": 24740580, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 616
    }
  },
  {
    "ph": "f", "id": 68, "pid": 1336760, "tid": 1381209, "ts": 1514656758255.068,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758288.255, "dur": 7.596,
    "args": {
      "External id": 2483818,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758292.802, "dur": 2.754,
    "args": {
      "External id": 2483819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758299.357, "dur": 7.301,
    "args": {
      "External id": 2483820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758301.474, "dur": 4.588,
    "args": {
      "External id": 2483821,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758305.034, "dur": 0.824,
    "args": {
      "External id": 2483822,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758312.057, "dur": 89.092,
    "args": {
      "External id": 2483823,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758312.812, "dur": 4.470,
    "args": {
      "External id": 2483824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758313.369, "dur": 3.505,
    "args": {
      "External id": 2483825,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758314.039, "dur": 2.748,
    "args": {
      "External id": 2483826,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758318.450, "dur": 82.197,
    "args": {
      "External id": 2483827,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758319.521, "dur": 80.300,
    "args": {
      "External id": 2483828,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758404.959, "dur": 4.483,
    "args": {
      "External id": 2483829,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758407.861, "dur": 1.446,
    "args": {
      "External id": 2483830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758443.320, "dur": 4.754,
    "args": {
      "External id": 2483831,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758449.120, "dur": 2.111,
    "args": {
      "External id": 2483832,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758451.968, "dur": 2.007,
    "args": {
      "External id": 2483833,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758484.773, "dur": 1.985,
    "args": {
      "External id": 2483834,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758485.485, "dur": 1.135,
    "args": {
      "External id": 2483835,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758511.671, "dur": 153.212,
    "args": {
      "External id": 2483836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758517.262, "dur": 4.382,
    "args": {
      "External id": 2483837,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758520.153, "dur": 0.706,
    "args": {
      "External id": 2483838,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758523.269, "dur": 29.171,
    "args": {
      "External id": 2483839,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758548.166, "dur": 2.836,
    "args": {
      "External id": 2483840,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758554.126, "dur": 4.078,
    "args": {
      "External id": 2483841,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758557.226, "dur": 0.618,
    "args": {
      "External id": 2483842,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758558.924, "dur": 4.060,
    "args": {
      "External id": 2483843,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758561.975, "dur": 0.442,
    "args": {
      "External id": 2483844,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758566.921, "dur": 2.334,
    "args": {
      "External id": 2483845,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758568.284, "dur": 0.686,
    "args": {
      "External id": 2483846,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758575.462, "dur": 5.106,
    "args": {
      "External id": 2483847,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758578.522, "dur": 1.852,
    "args": {
      "External id": 2483848,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758582.958, "dur": 1.559,
    "args": {
      "External id": 2483849,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758583.890, "dur": 0.371,
    "args": {
      "External id": 2483850,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758585.000, "dur": 5.560,
    "args": {
      "External id": 2483851,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758587.560, "dur": 2.917,
    "args": {
      "External id": 2483852,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758591.985, "dur": 60.062,
    "args": {
      "External id": 2483853,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758653.737, "dur": 2.988,
    "args": {
      "External id": 2483854,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758657.291, "dur": 3.360,
    "args": {
      "External id": 2483855,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758659.719, "dur": 0.458,
    "args": {
      "External id": 2483856,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758662.974, "dur": 0.863,
    "args": {
      "External id": 2483857,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758711.166, "dur": 7.895,
    "args": {
      "External id": 2483858,"Record function id": 0, "Ev Idx": 657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758713.037, "dur": 5.355,
    "args": {
      "External id": 2483859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758715.027, "dur": 2.700,
    "args": {
      "External id": 2483860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758715.830, "dur": 1.811,
    "args": {
      "External id": 2483861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758722.816, "dur": 6.896,
    "args": {
      "External id": 2483862,"Record function id": 0, "Sequence number": 24740579, "Fwd thread id": 1, "Ev Idx": 661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758723.946, "dur": 3.426,
    "args": {
      "External id": 2483863,"Sequence number": 24740579, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 662
    }
  },
  {
    "ph": "f", "id": 69, "pid": 1336760, "tid": 1381209, "ts": 1514656758723.946,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758725.450, "dur": 1.676,
    "args": {
      "External id": 2483864,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758726.085, "dur": 0.911,
    "args": {
      "External id": 2483865,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758733.185, "dur": 124.584,
    "args": {
      "External id": 2483866,"Record function id": 0, "Sequence number": 24740578, "Fwd thread id": 1, "Ev Idx": 665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758734.536, "dur": 115.965,
    "args": {
      "External id": 2483867,"Sequence number": 24740578, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 666
    }
  },
  {
    "ph": "f", "id": 70, "pid": 1336760, "tid": 1381209, "ts": 1514656758734.536,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758741.031, "dur": 5.630,
    "args": {
      "External id": 2483868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758742.386, "dur": 3.709,
    "args": {
      "External id": 2483869,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758745.365, "dur": 0.569,
    "args": {
      "External id": 2483870,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758747.888, "dur": 41.225,
    "args": {
      "External id": 2483871,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758790.347, "dur": 5.959,
    "args": {
      "External id": 2483872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758791.076, "dur": 4.371,
    "args": {
      "External id": 2483873,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758794.336, "dur": 0.940,
    "args": {
      "External id": 2483874,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758797.915, "dur": 2.501,
    "args": {
      "External id": 2483875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758798.853, "dur": 1.118,
    "args": {
      "External id": 2483876,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758799.583, "dur": 0.321,
    "args": {
      "External id": 2483877,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758800.990, "dur": 48.704,
    "args": {
      "External id": 2483878,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758862.361, "dur": 7.706,
    "args": {
      "External id": 2483879,"Record function id": 0, "Sequence number": 24740577, "Fwd thread id": 1, "Ev Idx": 678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758863.358, "dur": 5.189,
    "args": {
      "External id": 2483880,"Sequence number": 24740577, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 679
    }
  },
  {
    "ph": "f", "id": 71, "pid": 1336760, "tid": 1381209, "ts": 1514656758863.358,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758866.753, "dur": 1.645,
    "args": {
      "External id": 2483881,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758867.267, "dur": 1.006,
    "args": {
      "External id": 2483882,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758873.703, "dur": 9.539,
    "args": {
      "External id": 2483883,"Record function id": 0, "Sequence number": 24740576, "Fwd thread id": 1, "Ev Idx": 682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758876.633, "dur": 3.509,
    "args": {
      "External id": 2483884,"Sequence number": 24740576, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 683
    }
  },
  {
    "ph": "f", "id": 72, "pid": 1336760, "tid": 1381209, "ts": 1514656758876.633,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758877.636, "dur": 2.290,
    "args": {
      "External id": 2483885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758878.196, "dur": 1.257,
    "args": {
      "External id": 2483886,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758878.978, "dur": 0.373,
    "args": {
      "External id": 2483887,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758886.972, "dur": 4.293,
    "args": {
      "External id": 2483888,"Record function id": 0, "Ev Idx": 687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758887.981, "dur": 2.807,
    "args": {
      "External id": 2483889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758888.965, "dur": 1.480,
    "args": {
      "External id": 2483890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758889.438, "dur": 0.928,
    "args": {
      "External id": 2483891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758894.292, "dur": 9.228,
    "args": {
      "External id": 2483892,"Record function id": 0, "Sequence number": 24740575, "Fwd thread id": 1, "Ev Idx": 691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758896.798, "dur": 4.429,
    "args": {
      "External id": 2483893,"Sequence number": 24740575, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 692
    }
  },
  {
    "ph": "f", "id": 73, "pid": 1336760, "tid": 1381209, "ts": 1514656758896.798,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758898.067, "dur": 3.009,
    "args": {
      "External id": 2483894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758900.282, "dur": 0.684,
    "args": {
      "External id": 2483895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758906.615, "dur": 161.075,
    "args": {
      "External id": 2483896,"Record function id": 0, "Sequence number": 24740574, "Fwd thread id": 1, "Ev Idx": 695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758907.348, "dur": 148.302,
    "args": {
      "External id": 2483897,"Sequence number": 24740574, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 696
    }
  },
  {
    "ph": "f", "id": 74, "pid": 1336760, "tid": 1381209, "ts": 1514656758907.348,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758909.964, "dur": 5.023,
    "args": {
      "External id": 2483898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758910.520, "dur": 4.039,
    "args": {
      "External id": 2483899,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758914.053, "dur": 0.418,
    "args": {
      "External id": 2483900,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758917.664, "dur": 35.527,
    "args": {
      "External id": 2483901,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758954.217, "dur": 4.801,
    "args": {
      "External id": 2483902,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758954.821, "dur": 3.691,
    "args": {
      "External id": 2483903,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758955.721, "dur": 2.618,
    "args": {
      "External id": 2483904,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758960.408, "dur": 9.376,
    "args": {
      "External id": 2483905,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758961.079, "dur": 8.004,
    "args": {
      "External id": 2483906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758967.052, "dur": 1.943,
    "args": {
      "External id": 2483907,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656758970.285, "dur": 84.169,
    "args": {
      "External id": 2483908,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759075.565, "dur": 33.957,
    "args": {
      "External id": 2483909,"Record function id": 0, "Sequence number": 24740573, "Fwd thread id": 1, "Ev Idx": 708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759076.832, "dur": 4.782,
    "args": {
      "External id": 2483910,"Sequence number": 24740573, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 709
    }
  },
  {
    "ph": "f", "id": 75, "pid": 1336760, "tid": 1381209, "ts": 1514656759076.832,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759078.571, "dur": 2.904,
    "args": {
      "External id": 2483911,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759079.536, "dur": 1.687,
    "args": {
      "External id": 2483912,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759084.996, "dur": 21.398,
    "args": {
      "External id": 2483913,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759113.142, "dur": 11.101,
    "args": {
      "External id": 2483914,"Record function id": 0, "Sequence number": 24740572, "Fwd thread id": 1, "Ev Idx": 713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759114.435, "dur": 7.608,
    "args": {
      "External id": 2483915,"Sequence number": 24740572, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 714
    }
  },
  {
    "ph": "f", "id": 76, "pid": 1336760, "tid": 1381209, "ts": 1514656759114.435,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759115.346, "dur": 6.488,
    "args": {
      "External id": 2483916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759116.278, "dur": 5.059,
    "args": {
      "External id": 2483917,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759120.666, "dur": 0.567,
    "args": {
      "External id": 2483918,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759128.412, "dur": 5.876,
    "args": {
      "External id": 2483919,"Record function id": 0, "Ev Idx": 718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759129.825, "dur": 3.969,
    "args": {
      "External id": 2483920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759131.152, "dur": 2.213,
    "args": {
      "External id": 2483921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759131.767, "dur": 1.511,
    "args": {
      "External id": 2483922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759139.933, "dur": 423.594,
    "args": {
      "External id": 2483923,"Record function id": 0, "Sequence number": 24740571, "Fwd thread id": 1, "Ev Idx": 722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759141.096, "dur": 367.815,
    "args": {
      "External id": 2483924,"Sequence number": 24740571, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 723
    }
  },
  {
    "ph": "f", "id": 77, "pid": 1336760, "tid": 1381209, "ts": 1514656759141.096,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759164.966, "dur": 34.687,
    "args": {
      "External id": 2483925,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759166.165, "dur": 33.265,
    "args": {
      "External id": 2483926,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759168.776, "dur": 8.382,
    "args": {
      "External id": 2483927,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759173.314, "dur": 3.300,
    "args": {
      "External id": 2483928,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759178.429, "dur": 20.496,
    "args": {
      "External id": 2483929,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759211.616, "dur": 3.769,
    "args": {
      "External id": 2483930,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759214.239, "dur": 1.003,
    "args": {
      "External id": 2483931,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759218.987, "dur": 2.989,
    "args": {
      "External id": 2483932,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759219.704, "dur": 2.185,
    "args": {
      "External id": 2483933,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759233.986, "dur": 2.327,
    "args": {
      "External id": 2483934,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759247.017, "dur": 2.343,
    "args": {
      "External id": 2483935,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759408.129, "dur": 2.103,
    "args": {
      "External id": 2483936,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759414.179, "dur": 28.954,
    "args": {
      "External id": 2483937,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759422.207, "dur": 0.863,
    "args": {
      "External id": 2483938,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759448.158, "dur": 27.270,
    "args": {
      "External id": 2483939,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759449.633, "dur": 25.512,
    "args": {
      "External id": 2483940,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759454.904, "dur": 4.043,
    "args": {
      "External id": 2483941,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759460.220, "dur": 14.301,
    "args": {
      "External id": 2483942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759479.416, "dur": 4.427,
    "args": {
      "External id": 2483943,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759482.687, "dur": 1.011,
    "args": {
      "External id": 2483944,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759489.476, "dur": 2.442,
    "args": {
      "External id": 2483945,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759490.125, "dur": 1.634,
    "args": {
      "External id": 2483946,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759495.638, "dur": 4.366,
    "args": {
      "External id": 2483947,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759496.212, "dur": 3.709,
    "args": {
      "External id": 2483948,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759524.664, "dur": 36.320,
    "args": {
      "External id": 2483949,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759576.846, "dur": 10.997,
    "args": {
      "External id": 2483950,"Record function id": 0, "Ev Idx": 749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759579.018, "dur": 8.087,
    "args": {
      "External id": 2483951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759580.954, "dur": 5.404,
    "args": {
      "External id": 2483952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759583.931, "dur": 2.352,
    "args": {
      "External id": 2483953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759591.371, "dur": 8.853,
    "args": {
      "External id": 2483954,"Record function id": 0, "Sequence number": 24740570, "Fwd thread id": 1, "Ev Idx": 753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759592.433, "dur": 4.315,
    "args": {
      "External id": 2483955,"Sequence number": 24740570, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 754
    }
  },
  {
    "ph": "f", "id": 78, "pid": 1336760, "tid": 1381209, "ts": 1514656759592.433,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759594.300, "dur": 2.231,
    "args": {
      "External id": 2483956,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759594.922, "dur": 1.480,
    "args": {
      "External id": 2483957,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759603.599, "dur": 134.765,
    "args": {
      "External id": 2483958,"Record function id": 0, "Sequence number": 24740569, "Fwd thread id": 1, "Ev Idx": 757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759604.419, "dur": 126.488,
    "args": {
      "External id": 2483959,"Sequence number": 24740569, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 758
    }
  },
  {
    "ph": "f", "id": 79, "pid": 1336760, "tid": 1381209, "ts": 1514656759604.419,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759608.508, "dur": 9.342,
    "args": {
      "External id": 2483960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759612.283, "dur": 5.025,
    "args": {
      "External id": 2483961,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759616.108, "dur": 1.026,
    "args": {
      "External id": 2483962,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759619.012, "dur": 57.780,
    "args": {
      "External id": 2483963,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759678.182, "dur": 5.427,
    "args": {
      "External id": 2483964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759679.004, "dur": 4.056,
    "args": {
      "External id": 2483965,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759680.127, "dur": 2.786,
    "args": {
      "External id": 2483966,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759687.216, "dur": 3.117,
    "args": {
      "External id": 2483967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759688.481, "dur": 1.333,
    "args": {
      "External id": 2483968,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759689.276, "dur": 0.470,
    "args": {
      "External id": 2483969,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759691.059, "dur": 39.072,
    "args": {
      "External id": 2483970,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759743.430, "dur": 10.342,
    "args": {
      "External id": 2483971,"Record function id": 0, "Sequence number": 24740568, "Fwd thread id": 1, "Ev Idx": 770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759744.539, "dur": 6.608,
    "args": {
      "External id": 2483972,"Sequence number": 24740568, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 771
    }
  },
  {
    "ph": "f", "id": 80, "pid": 1336760, "tid": 1381209, "ts": 1514656759744.539,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759747.653, "dur": 3.340,
    "args": {
      "External id": 2483973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759748.125, "dur": 2.713,
    "args": {
      "External id": 2483974,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759759.541, "dur": 7.782,
    "args": {
      "External id": 2483975,"Record function id": 0, "Sequence number": 24740567, "Fwd thread id": 1, "Ev Idx": 774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759761.848, "dur": 3.948,
    "args": {
      "External id": 2483976,"Sequence number": 24740567, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 775
    }
  },
  {
    "ph": "f", "id": 81, "pid": 1336760, "tid": 1381209, "ts": 1514656759761.848,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759762.636, "dur": 2.939,
    "args": {
      "External id": 2483977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759763.504, "dur": 1.620,
    "args": {
      "External id": 2483978,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759764.301, "dur": 0.675,
    "args": {
      "External id": 2483979,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759771.113, "dur": 4.718,
    "args": {
      "External id": 2483980,"Record function id": 0, "Ev Idx": 779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759772.266, "dur": 3.053,
    "args": {
      "External id": 2483981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759773.422, "dur": 1.687,
    "args": {
      "External id": 2483982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759774.018, "dur": 0.960,
    "args": {
      "External id": 2483983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759778.837, "dur": 11.778,
    "args": {
      "External id": 2483984,"Record function id": 0, "Sequence number": 24740566, "Fwd thread id": 1, "Ev Idx": 783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759779.855, "dur": 8.099,
    "args": {
      "External id": 2483985,"Sequence number": 24740566, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 784
    }
  },
  {
    "ph": "f", "id": 82, "pid": 1336760, "tid": 1381209, "ts": 1514656759779.855,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759783.833, "dur": 3.969,
    "args": {
      "External id": 2483986,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759786.487, "dur": 1.167,
    "args": {
      "External id": 2483987,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759794.576, "dur": 323.683,
    "args": {
      "External id": 2483988,"Record function id": 0, "Sequence number": 24740565, "Fwd thread id": 1, "Ev Idx": 787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759795.542, "dur": 298.226,
    "args": {
      "External id": 2483989,"Sequence number": 24740565, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 788
    }
  },
  {
    "ph": "f", "id": 83, "pid": 1336760, "tid": 1381209, "ts": 1514656759795.542,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759812.789, "dur": 5.783,
    "args": {
      "External id": 2483990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759814.964, "dur": 3.225,
    "args": {
      "External id": 2483991,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759820.552, "dur": 4.448,
    "args": {
      "External id": 2483992,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759822.774, "dur": 2.041,
    "args": {
      "External id": 2483993,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759826.335, "dur": 4.861,
    "args": {
      "External id": 2483994,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759827.555, "dur": 3.469,
    "args": {
      "External id": 2483995,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759857.975, "dur": 208.451,
    "args": {
      "External id": 2483996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759936.621, "dur": 3.085,
    "args": {
      "External id": 2483997,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656759941.381, "dur": 3.836,
    "args": {
      "External id": 2483998,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760080.901, "dur": 3.815,
    "args": {
      "External id": 2483999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760087.625, "dur": 0.765,
    "args": {
      "External id": 2484000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760090.273, "dur": 0.713,
    "args": {
      "External id": 2484001,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760127.848, "dur": 228.869,
    "args": {
      "External id": 2484002,"Record function id": 0, "Sequence number": 24740564, "Fwd thread id": 1, "Ev Idx": 801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760129.601, "dur": 220.446,
    "args": {
      "External id": 2484003,"Sequence number": 24740564, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 802
    }
  },
  {
    "ph": "f", "id": 84, "pid": 1336760, "tid": 1381209, "ts": 1514656760129.601,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760153.244, "dur": 47.724,
    "args": {
      "External id": 2484004,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760156.202, "dur": 3.487,
    "args": {
      "External id": 2484005,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760161.129, "dur": 39.367,
    "args": {
      "External id": 2484006,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760211.245, "dur": 6.156,
    "args": {
      "External id": 2484007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760214.792, "dur": 2.253,
    "args": {
      "External id": 2484008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760363.710, "dur": 145.857,
    "args": {
      "External id": 2484009,"Record function id": 0, "Sequence number": 24740563, "Fwd thread id": 1, "Ev Idx": 808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760365.426, "dur": 137.749,
    "args": {
      "External id": 2484010,"Sequence number": 24740563, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 809
    }
  },
  {
    "ph": "f", "id": 85, "pid": 1336760, "tid": 1381209, "ts": 1514656760365.426,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760378.069, "dur": 29.532,
    "args": {
      "External id": 2484011,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760380.076, "dur": 2.463,
    "args": {
      "External id": 2484012,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760383.339, "dur": 23.877,
    "args": {
      "External id": 2484013,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760414.315, "dur": 7.383,
    "args": {
      "External id": 2484014,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760419.477, "dur": 1.900,
    "args": {
      "External id": 2484015,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760515.803, "dur": 32.549,
    "args": {
      "External id": 2484016,"Record function id": 0, "Sequence number": 24740562, "Fwd thread id": 1, "Ev Idx": 815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760517.154, "dur": 9.232,
    "args": {
      "External id": 2484017,"Sequence number": 24740562, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 816
    }
  },
  {
    "ph": "f", "id": 86, "pid": 1336760, "tid": 1381209, "ts": 1514656760517.154,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760519.226, "dur": 6.799,
    "args": {
      "External id": 2484018,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760520.567, "dur": 5.193,
    "args": {
      "External id": 2484019,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760554.340, "dur": 12.182,
    "args": {
      "External id": 2484020,"Record function id": 0, "Sequence number": 24740561, "Fwd thread id": 1, "Ev Idx": 819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760557.404, "dur": 6.245,
    "args": {
      "External id": 2484021,"Sequence number": 24740561, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 820
    }
  },
  {
    "ph": "f", "id": 87, "pid": 1336760, "tid": 1381209, "ts": 1514656760557.404,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760558.934, "dur": 4.547,
    "args": {
      "External id": 2484022,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760561.900, "dur": 1.440,
    "args": {
      "External id": 2484023,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760571.943, "dur": 6.088,
    "args": {
      "External id": 2484024,"Record function id": 0, "Sequence number": 24740560, "Fwd thread id": 1, "Ev Idx": 823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760572.756, "dur": 3.044,
    "args": {
      "External id": 2484025,"Sequence number": 24740560, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 824
    }
  },
  {
    "ph": "f", "id": 88, "pid": 1336760, "tid": 1381209, "ts": 1514656760572.756,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760573.964, "dur": 1.698,
    "args": {
      "External id": 2484026,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760574.751, "dur": 0.771,
    "args": {
      "External id": 2484027,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760581.453, "dur": 10.649,
    "args": {
      "External id": 2484028,"Record function id": 0, "Sequence number": 24740559, "Fwd thread id": 1, "Ev Idx": 827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760582.325, "dur": 6.735,
    "args": {
      "External id": 2484029,"Sequence number": 24740559, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 828
    }
  },
  {
    "ph": "f", "id": 89, "pid": 1336760, "tid": 1381209, "ts": 1514656760582.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760583.396, "dur": 5.514,
    "args": {
      "External id": 2484030,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760587.892, "dur": 0.893,
    "args": {
      "External id": 2484031,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760595.587, "dur": 151.532,
    "args": {
      "External id": 2484032,"Record function id": 0, "Sequence number": 24740558, "Fwd thread id": 1, "Ev Idx": 831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760596.544, "dur": 141.119,
    "args": {
      "External id": 2484033,"Sequence number": 24740558, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 832
    }
  },
  {
    "ph": "f", "id": 90, "pid": 1336760, "tid": 1381209, "ts": 1514656760596.544,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760600.944, "dur": 6.242,
    "args": {
      "External id": 2484034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760602.806, "dur": 3.735,
    "args": {
      "External id": 2484035,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760604.605, "dur": 1.609,
    "args": {
      "External id": 2484036,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760608.838, "dur": 68.353,
    "args": {
      "External id": 2484037,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760678.744, "dur": 6.459,
    "args": {
      "External id": 2484038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760681.339, "dur": 2.976,
    "args": {
      "External id": 2484039,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760682.752, "dur": 1.390,
    "args": {
      "External id": 2484040,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760687.024, "dur": 5.461,
    "args": {
      "External id": 2484041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760688.331, "dur": 3.326,
    "args": {
      "External id": 2484042,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760691.235, "dur": 0.309,
    "args": {
      "External id": 2484043,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760692.987, "dur": 43.876,
    "args": {
      "External id": 2484044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760752.277, "dur": 9.486,
    "args": {
      "External id": 2484045,"Record function id": 0, "Sequence number": 24740557, "Fwd thread id": 1, "Ev Idx": 844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760753.348, "dur": 5.951,
    "args": {
      "External id": 2484046,"Sequence number": 24740557, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 845
    }
  },
  {
    "ph": "f", "id": 91, "pid": 1336760, "tid": 1381209, "ts": 1514656760753.348,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760757.086, "dur": 2.084,
    "args": {
      "External id": 2484047,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760757.938, "dur": 1.067,
    "args": {
      "External id": 2484048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760765.218, "dur": 7.322,
    "args": {
      "External id": 2484049,"Record function id": 0, "Sequence number": 24740556, "Fwd thread id": 1, "Ev Idx": 848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760766.244, "dur": 3.482,
    "args": {
      "External id": 2484050,"Sequence number": 24740556, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 849
    }
  },
  {
    "ph": "f", "id": 92, "pid": 1336760, "tid": 1381209, "ts": 1514656760766.244,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760766.958, "dur": 2.565,
    "args": {
      "External id": 2484051,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760767.481, "dur": 1.567,
    "args": {
      "External id": 2484052,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760768.399, "dur": 0.498,
    "args": {
      "External id": 2484053,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760780.608, "dur": 12.112,
    "args": {
      "External id": 2484054,"Record function id": 0, "Ev Idx": 853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760782.569, "dur": 9.263,
    "args": {
      "External id": 2484055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760785.026, "dur": 6.378,
    "args": {
      "External id": 2484056,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760788.674, "dur": 2.645,
    "args": {
      "External id": 2484057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760795.896, "dur": 7.331,
    "args": {
      "External id": 2484058,"Record function id": 0, "Sequence number": 24740555, "Fwd thread id": 1, "Ev Idx": 857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760797.179, "dur": 3.097,
    "args": {
      "External id": 2484059,"Sequence number": 24740555, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 858
    }
  },
  {
    "ph": "f", "id": 93, "pid": 1336760, "tid": 1381209, "ts": 1514656760797.179,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760798.303, "dur": 1.835,
    "args": {
      "External id": 2484060,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760799.183, "dur": 0.852,
    "args": {
      "External id": 2484061,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760806.342, "dur": 90.460,
    "args": {
      "External id": 2484062,"Record function id": 0, "Sequence number": 24740554, "Fwd thread id": 1, "Ev Idx": 861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760807.283, "dur": 83.817,
    "args": {
      "External id": 2484063,"Sequence number": 24740554, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 862
    }
  },
  {
    "ph": "f", "id": 94, "pid": 1336760, "tid": 1381209, "ts": 1514656760807.283,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760811.160, "dur": 4.494,
    "args": {
      "External id": 2484064,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760814.006, "dur": 1.236,
    "args": {
      "External id": 2484065,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760814.687, "dur": 0.453,
    "args": {
      "External id": 2484066,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760816.365, "dur": 25.989,
    "args": {
      "External id": 2484067,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760843.416, "dur": 5.135,
    "args": {
      "External id": 2484068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760844.330, "dur": 3.706,
    "args": {
      "External id": 2484069,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760846.881, "dur": 1.049,
    "args": {
      "External id": 2484070,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760852.114, "dur": 4.788,
    "args": {
      "External id": 2484071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760853.189, "dur": 3.195,
    "args": {
      "External id": 2484072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760854.157, "dur": 2.164,
    "args": {
      "External id": 2484073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760857.474, "dur": 32.808,
    "args": {
      "External id": 2484074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760901.200, "dur": 34.391,
    "args": {
      "External id": 2484075,"Record function id": 0, "Sequence number": 24740553, "Fwd thread id": 1, "Ev Idx": 874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760902.174, "dur": 5.716,
    "args": {
      "External id": 2484076,"Sequence number": 24740553, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 875
    }
  },
  {
    "ph": "f", "id": 95, "pid": 1336760, "tid": 1381209, "ts": 1514656760902.174,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760903.873, "dur": 3.887,
    "args": {
      "External id": 2484077,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760906.489, "dur": 1.160,
    "args": {
      "External id": 2484078,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760911.114, "dur": 20.448,
    "args": {
      "External id": 2484079,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760939.505, "dur": 7.680,
    "args": {
      "External id": 2484080,"Record function id": 0, "Sequence number": 24740552, "Fwd thread id": 1, "Ev Idx": 879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760940.572, "dur": 4.213,
    "args": {
      "External id": 2484081,"Sequence number": 24740552, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 880
    }
  },
  {
    "ph": "f", "id": 96, "pid": 1336760, "tid": 1381209, "ts": 1514656760940.572,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760941.705, "dur": 2.886,
    "args": {
      "External id": 2484082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760942.451, "dur": 1.679,
    "args": {
      "External id": 2484083,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760943.431, "dur": 0.581,
    "args": {
      "External id": 2484084,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760951.098, "dur": 6.606,
    "args": {
      "External id": 2484085,"Record function id": 0, "Ev Idx": 884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760952.508, "dur": 4.734,
    "args": {
      "External id": 2484086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760953.400, "dur": 3.424,
    "args": {
      "External id": 2484087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760955.773, "dur": 0.938,
    "args": {
      "External id": 2484088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760960.905, "dur": 8.677,
    "args": {
      "External id": 2484089,"Record function id": 0, "Sequence number": 24740551, "Fwd thread id": 1, "Ev Idx": 888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760961.896, "dur": 4.838,
    "args": {
      "External id": 2484090,"Sequence number": 24740551, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 889
    }
  },
  {
    "ph": "f", "id": 97, "pid": 1336760, "tid": 1381209, "ts": 1514656760961.896,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760963.069, "dur": 3.524,
    "args": {
      "External id": 2484091,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760965.617, "dur": 0.890,
    "args": {
      "External id": 2484092,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760974.644, "dur": 143.432,
    "args": {
      "External id": 2484093,"Record function id": 0, "Sequence number": 24740550, "Fwd thread id": 1, "Ev Idx": 892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760975.474, "dur": 133.195,
    "args": {
      "External id": 2484094,"Sequence number": 24740550, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 893
    }
  },
  {
    "ph": "f", "id": 98, "pid": 1336760, "tid": 1381209, "ts": 1514656760975.474,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760978.621, "dur": 4.846,
    "args": {
      "External id": 2484095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760979.169, "dur": 3.856,
    "args": {
      "External id": 2484096,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656760982.255, "dur": 0.613,
    "args": {
      "External id": 2484097,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761020.121, "dur": 35.883,
    "args": {
      "External id": 2484098,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761057.946, "dur": 4.214,
    "args": {
      "External id": 2484099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761058.835, "dur": 2.646,
    "args": {
      "External id": 2484100,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761060.123, "dur": 1.243,
    "args": {
      "External id": 2484101,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761063.455, "dur": 9.048,
    "args": {
      "External id": 2484102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761068.784, "dur": 3.275,
    "args": {
      "External id": 2484103,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761071.500, "dur": 0.492,
    "args": {
      "External id": 2484104,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761073.180, "dur": 34.857,
    "args": {
      "External id": 2484105,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761124.313, "dur": 25.539,
    "args": {
      "External id": 2484106,"Record function id": 0, "Sequence number": 24740549, "Fwd thread id": 1, "Ev Idx": 905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761125.304, "dur": 3.730,
    "args": {
      "External id": 2484107,"Sequence number": 24740549, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 906
    }
  },
  {
    "ph": "f", "id": 99, "pid": 1336760, "tid": 1381209, "ts": 1514656761125.304,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761126.796, "dur": 2.089,
    "args": {
      "External id": 2484108,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761127.568, "dur": 1.223,
    "args": {
      "External id": 2484109,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761131.590, "dur": 15.796,
    "args": {
      "External id": 2484110,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761153.761, "dur": 12.198,
    "args": {
      "External id": 2484111,"Record function id": 0, "Sequence number": 24740548, "Fwd thread id": 1, "Ev Idx": 910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761157.264, "dur": 6.502,
    "args": {
      "External id": 2484112,"Sequence number": 24740548, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 911
    }
  },
  {
    "ph": "f", "id": 100, "pid": 1336760, "tid": 1381209, "ts": 1514656761157.264,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761158.429, "dur": 5.135,
    "args": {
      "External id": 2484113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761158.948, "dur": 4.124,
    "args": {
      "External id": 2484114,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761162.236, "dur": 0.692,
    "args": {
      "External id": 2484115,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761170.327, "dur": 5.596,
    "args": {
      "External id": 2484116,"Record function id": 0, "Ev Idx": 915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761171.852, "dur": 3.608,
    "args": {
      "External id": 2484117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761173.171, "dur": 1.934,
    "args": {
      "External id": 2484118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761173.720, "dur": 1.283,
    "args": {
      "External id": 2484119,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761180.275, "dur": 375.364,
    "args": {
      "External id": 2484120,"Record function id": 0, "Sequence number": 24740547, "Fwd thread id": 1, "Ev Idx": 919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761181.680, "dur": 323.551,
    "args": {
      "External id": 2484121,"Sequence number": 24740547, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 920
    }
  },
  {
    "ph": "f", "id": 101, "pid": 1336760, "tid": 1381209, "ts": 1514656761181.680,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761215.863, "dur": 4.264,
    "args": {
      "External id": 2484122,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761218.930, "dur": 1.041,
    "args": {
      "External id": 2484123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761234.077, "dur": 5.719,
    "args": {
      "External id": 2484124,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761248.799, "dur": 1.844,
    "args": {
      "External id": 2484125,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761403.709, "dur": 1.381,
    "args": {
      "External id": 2484126,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761409.292, "dur": 33.158,
    "args": {
      "External id": 2484127,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761418.894, "dur": 1.017,
    "args": {
      "External id": 2484128,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761451.223, "dur": 32.720,
    "args": {
      "External id": 2484129,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761453.126, "dur": 30.550,
    "args": {
      "External id": 2484130,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761460.154, "dur": 6.225,
    "args": {
      "External id": 2484131,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761467.830, "dur": 15.351,
    "args": {
      "External id": 2484132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761488.461, "dur": 2.573,
    "args": {
      "External id": 2484133,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761489.543, "dur": 1.345,
    "args": {
      "External id": 2484134,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761496.865, "dur": 1.758,
    "args": {
      "External id": 2484135,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761497.555, "dur": 0.983,
    "args": {
      "External id": 2484136,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761517.558, "dur": 32.943,
    "args": {
      "External id": 2484137,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761567.992, "dur": 11.339,
    "args": {
      "External id": 2484138,"Record function id": 0, "Ev Idx": 937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761572.102, "dur": 6.443,
    "args": {
      "External id": 2484139,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761574.170, "dur": 3.395,
    "args": {
      "External id": 2484140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761575.207, "dur": 2.269,
    "args": {
      "External id": 2484141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761582.911, "dur": 6.683,
    "args": {
      "External id": 2484142,"Record function id": 0, "Sequence number": 24740546, "Fwd thread id": 1, "Ev Idx": 941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761584.405, "dur": 1.273,
    "args": {
      "External id": 2484143,"Sequence number": 24740546, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 942
    }
  },
  {
    "ph": "f", "id": 102, "pid": 1336760, "tid": 1381209, "ts": 1514656761584.405,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761593.942, "dur": 485.477,
    "args": {
      "External id": 2484144,"Record function id": 0, "Sequence number": 24740545, "Fwd thread id": 1, "Ev Idx": 943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761595.347, "dur": 470.201,
    "args": {
      "External id": 2484145,"Sequence number": 24740545, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 944
    }
  },
  {
    "ph": "f", "id": 103, "pid": 1336760, "tid": 1381209, "ts": 1514656761595.347,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761630.598, "dur": 8.513,
    "args": {
      "External id": 2484146,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761635.498, "dur": 3.360,
    "args": {
      "External id": 2484147,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761642.372, "dur": 9.117,
    "args": {
      "External id": 2484148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761643.840, "dur": 7.084,
    "args": {
      "External id": 2484149,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761647.411, "dur": 3.363,
    "args": {
      "External id": 2484150,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761655.222, "dur": 97.577,
    "args": {
      "External id": 2484151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761655.960, "dur": 4.270,
    "args": {
      "External id": 2484152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761658.311, "dur": 1.490,
    "args": {
      "External id": 2484153,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761659.125, "dur": 0.551,
    "args": {
      "External id": 2484154,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761661.622, "dur": 90.612,
    "args": {
      "External id": 2484155,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761663.156, "dur": 88.416,
    "args": {
      "External id": 2484156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761756.885, "dur": 2.759,
    "args": {
      "External id": 2484157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761758.170, "dur": 1.340,
    "args": {
      "External id": 2484158,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761791.639, "dur": 5.604,
    "args": {
      "External id": 2484159,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761800.529, "dur": 2.100,
    "args": {
      "External id": 2484160,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761803.321, "dur": 2.204,
    "args": {
      "External id": 2484161,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761835.925, "dur": 2.242,
    "args": {
      "External id": 2484162,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761836.825, "dur": 1.160,
    "args": {
      "External id": 2484163,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761860.808, "dur": 183.262,
    "args": {
      "External id": 2484164,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761865.865, "dur": 8.127,
    "args": {
      "External id": 2484165,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761870.543, "dur": 2.783,
    "args": {
      "External id": 2484166,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761875.465, "dur": 7.701,
    "args": {
      "External id": 2484167,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761881.758, "dur": 0.587,
    "args": {
      "External id": 2484168,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761884.535, "dur": 1.709,
    "args": {
      "External id": 2484169,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761885.420, "dur": 0.534,
    "args": {
      "External id": 2484170,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761889.000, "dur": 1.798,
    "args": {
      "External id": 2484171,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761889.862, "dur": 0.510,
    "args": {
      "External id": 2484172,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761896.413, "dur": 2.058,
    "args": {
      "External id": 2484173,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761897.595, "dur": 0.591,
    "args": {
      "External id": 2484174,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761899.506, "dur": 6.284,
    "args": {
      "External id": 2484175,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761903.793, "dur": 1.840,
    "args": {
      "External id": 2484176,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761906.546, "dur": 3.184,
    "args": {
      "External id": 2484177,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761909.157, "dur": 0.321,
    "args": {
      "External id": 2484178,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761912.022, "dur": 3.716,
    "args": {
      "External id": 2484179,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761912.503, "dur": 3.146,
    "args": {
      "External id": 2484180,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656761916.785, "dur": 109.377,
    "args": {
      "External id": 2484181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762032.075, "dur": 2.171,
    "args": {
      "External id": 2484182,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762034.815, "dur": 4.063,
    "args": {
      "External id": 2484183,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762037.875, "dur": 0.452,
    "args": {
      "External id": 2484184,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762041.719, "dur": 1.029,
    "args": {
      "External id": 2484185,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762091.113, "dur": 9.508,
    "args": {
      "External id": 2484186,"Record function id": 0, "Ev Idx": 985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762093.662, "dur": 6.339,
    "args": {
      "External id": 2484187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762095.958, "dur": 3.274,
    "args": {
      "External id": 2484188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762097.022, "dur": 2.109,
    "args": {
      "External id": 2484189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762104.448, "dur": 9.673,
    "args": {
      "External id": 2484190,"Record function id": 0, "Sequence number": 24740544, "Fwd thread id": 1, "Ev Idx": 989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762105.495, "dur": 5.362,
    "args": {
      "External id": 2484191,"Sequence number": 24740544, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 990
    }
  },
  {
    "ph": "f", "id": 104, "pid": 1336760, "tid": 1381209, "ts": 1514656762105.495,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762108.914, "dur": 1.765,
    "args": {
      "External id": 2484192,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762109.494, "dur": 1.044,
    "args": {
      "External id": 2484193,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762117.838, "dur": 155.795,
    "args": {
      "External id": 2484194,"Record function id": 0, "Sequence number": 24740543, "Fwd thread id": 1, "Ev Idx": 993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762118.859, "dur": 147.465,
    "args": {
      "External id": 2484195,"Sequence number": 24740543, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 994
    }
  },
  {
    "ph": "f", "id": 105, "pid": 1336760, "tid": 1381209, "ts": 1514656762118.859,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762123.007, "dur": 5.959,
    "args": {
      "External id": 2484196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762124.119, "dur": 4.326,
    "args": {
      "External id": 2484197,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762127.700, "dur": 0.572,
    "args": {
      "External id": 2484198,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762130.197, "dur": 62.260,
    "args": {
      "External id": 2484199,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762193.615, "dur": 5.588,
    "args": {
      "External id": 2484200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762194.314, "dur": 4.156,
    "args": {
      "External id": 2484201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762197.476, "dur": 0.859,
    "args": {
      "External id": 2484202,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762200.956, "dur": 6.965,
    "args": {
      "External id": 2484203,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762202.494, "dur": 4.875,
    "args": {
      "External id": 2484204,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762205.049, "dur": 2.238,
    "args": {
      "External id": 2484205,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762208.378, "dur": 57.177,
    "args": {
      "External id": 2484206,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762278.761, "dur": 7.090,
    "args": {
      "External id": 2484207,"Record function id": 0, "Sequence number": 24740542, "Fwd thread id": 1, "Ev Idx": 1006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762279.741, "dur": 4.885,
    "args": {
      "External id": 2484208,"Sequence number": 24740542, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1007
    }
  },
  {
    "ph": "f", "id": 106, "pid": 1336760, "tid": 1381209, "ts": 1514656762279.741,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762281.020, "dur": 3.450,
    "args": {
      "External id": 2484209,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762283.385, "dur": 0.967,
    "args": {
      "External id": 2484210,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762289.064, "dur": 8.594,
    "args": {
      "External id": 2484211,"Record function id": 0, "Sequence number": 24740541, "Fwd thread id": 1, "Ev Idx": 1010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762290.102, "dur": 5.726,
    "args": {
      "External id": 2484212,"Sequence number": 24740541, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1011
    }
  },
  {
    "ph": "f", "id": 107, "pid": 1336760, "tid": 1381209, "ts": 1514656762290.102,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762290.990, "dur": 4.626,
    "args": {
      "External id": 2484213,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762291.543, "dur": 3.637,
    "args": {
      "External id": 2484214,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762294.458, "dur": 0.584,
    "args": {
      "External id": 2484215,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762301.503, "dur": 5.536,
    "args": {
      "External id": 2484216,"Record function id": 0, "Ev Idx": 1015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762302.771, "dur": 3.758,
    "args": {
      "External id": 2484217,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762303.902, "dur": 2.327,
    "args": {
      "External id": 2484218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762304.766, "dur": 1.370,
    "args": {
      "External id": 2484219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762312.038, "dur": 6.979,
    "args": {
      "External id": 2484220,"Record function id": 0, "Sequence number": 24740540, "Fwd thread id": 1, "Ev Idx": 1019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762312.927, "dur": 3.824,
    "args": {
      "External id": 2484221,"Sequence number": 24740540, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1020
    }
  },
  {
    "ph": "f", "id": 108, "pid": 1336760, "tid": 1381209, "ts": 1514656762312.927,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762313.924, "dur": 2.670,
    "args": {
      "External id": 2484222,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762315.863, "dur": 0.618,
    "args": {
      "External id": 2484223,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762322.198, "dur": 138.647,
    "args": {
      "External id": 2484224,"Record function id": 0, "Sequence number": 24740539, "Fwd thread id": 1, "Ev Idx": 1023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762348.042, "dur": 104.287,
    "args": {
      "External id": 2484225,"Sequence number": 24740539, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1024
    }
  },
  {
    "ph": "f", "id": 109, "pid": 1336760, "tid": 1381209, "ts": 1514656762348.042,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762353.575, "dur": 2.154,
    "args": {
      "External id": 2484226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762354.071, "dur": 1.244,
    "args": {
      "External id": 2484227,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762354.814, "dur": 0.397,
    "args": {
      "External id": 2484228,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762356.442, "dur": 33.255,
    "args": {
      "External id": 2484229,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762390.726, "dur": 8.051,
    "args": {
      "External id": 2484230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762391.141, "dur": 7.117,
    "args": {
      "External id": 2484231,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762393.955, "dur": 4.165,
    "args": {
      "External id": 2484232,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762399.794, "dur": 4.560,
    "args": {
      "External id": 2484233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762401.062, "dur": 2.928,
    "args": {
      "External id": 2484234,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762403.473, "dur": 0.453,
    "args": {
      "External id": 2484235,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762404.817, "dur": 46.772,
    "args": {
      "External id": 2484236,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762465.493, "dur": 36.695,
    "args": {
      "External id": 2484237,"Record function id": 0, "Sequence number": 24740538, "Fwd thread id": 1, "Ev Idx": 1036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762466.752, "dur": 5.047,
    "args": {
      "External id": 2484238,"Sequence number": 24740538, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1037
    }
  },
  {
    "ph": "f", "id": 110, "pid": 1336760, "tid": 1381209, "ts": 1514656762466.752,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762469.921, "dur": 1.724,
    "args": {
      "External id": 2484239,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762470.630, "dur": 0.913,
    "args": {
      "External id": 2484240,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762474.623, "dur": 19.365,
    "args": {
      "External id": 2484241,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762505.829, "dur": 8.912,
    "args": {
      "External id": 2484242,"Record function id": 0, "Sequence number": 24740537, "Fwd thread id": 1, "Ev Idx": 1041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762506.802, "dur": 5.724,
    "args": {
      "External id": 2484243,"Sequence number": 24740537, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1042
    }
  },
  {
    "ph": "f", "id": 111, "pid": 1336760, "tid": 1381209, "ts": 1514656762506.802,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762507.483, "dur": 4.852,
    "args": {
      "External id": 2484244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762508.206, "dur": 3.639,
    "args": {
      "External id": 2484245,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762511.248, "dur": 0.492,
    "args": {
      "External id": 2484246,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762518.483, "dur": 6.468,
    "args": {
      "External id": 2484247,"Record function id": 0, "Ev Idx": 1046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762519.766, "dur": 4.706,
    "args": {
      "External id": 2484248,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762520.661, "dur": 3.360,
    "args": {
      "External id": 2484249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762522.894, "dur": 1.003,
    "args": {
      "External id": 2484250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762545.365, "dur": 404.131,
    "args": {
      "External id": 2484251,"Record function id": 0, "Sequence number": 24740536, "Fwd thread id": 1, "Ev Idx": 1050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762547.288, "dur": 369.754,
    "args": {
      "External id": 2484252,"Sequence number": 24740536, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1051
    }
  },
  {
    "ph": "f", "id": 112, "pid": 1336760, "tid": 1381209, "ts": 1514656762547.288,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762570.810, "dur": 35.142,
    "args": {
      "External id": 2484253,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762572.326, "dur": 33.428,
    "args": {
      "External id": 2484254,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762575.229, "dur": 6.798,
    "args": {
      "External id": 2484255,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762577.785, "dur": 3.673,
    "args": {
      "External id": 2484256,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762583.289, "dur": 22.002,
    "args": {
      "External id": 2484257,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762620.183, "dur": 4.468,
    "args": {
      "External id": 2484258,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762620.839, "dur": 3.677,
    "args": {
      "External id": 2484259,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762628.137, "dur": 2.968,
    "args": {
      "External id": 2484260,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762630.359, "dur": 0.655,
    "args": {
      "External id": 2484261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762642.549, "dur": 2.291,
    "args": {
      "External id": 2484262,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762655.047, "dur": 1.820,
    "args": {
      "External id": 2484263,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762811.134, "dur": 2.327,
    "args": {
      "External id": 2484264,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762819.507, "dur": 29.081,
    "args": {
      "External id": 2484265,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762827.619, "dur": 0.731,
    "args": {
      "External id": 2484266,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762853.987, "dur": 29.888,
    "args": {
      "External id": 2484267,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762857.867, "dur": 25.723,
    "args": {
      "External id": 2484268,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762863.166, "dur": 4.462,
    "args": {
      "External id": 2484269,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762869.034, "dur": 14.054,
    "args": {
      "External id": 2484270,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762888.320, "dur": 2.167,
    "args": {
      "External id": 2484271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762889.339, "dur": 1.010,
    "args": {
      "External id": 2484272,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762896.622, "dur": 5.499,
    "args": {
      "External id": 2484273,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762898.962, "dur": 3.033,
    "args": {
      "External id": 2484274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762904.010, "dur": 3.639,
    "args": {
      "External id": 2484275,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762906.397, "dur": 1.167,
    "args": {
      "External id": 2484276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762932.229, "dur": 15.789,
    "args": {
      "External id": 2484277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762960.242, "dur": 7.839,
    "args": {
      "External id": 2484278,"Record function id": 0, "Ev Idx": 1077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762962.109, "dur": 5.328,
    "args": {
      "External id": 2484279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762963.938, "dur": 2.716,
    "args": {
      "External id": 2484280,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762964.911, "dur": 1.625,
    "args": {
      "External id": 2484281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762972.000, "dur": 9.154,
    "args": {
      "External id": 2484282,"Record function id": 0, "Sequence number": 24740535, "Fwd thread id": 1, "Ev Idx": 1081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762973.042, "dur": 5.363,
    "args": {
      "External id": 2484283,"Sequence number": 24740535, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1082
    }
  },
  {
    "ph": "f", "id": 113, "pid": 1336760, "tid": 1381209, "ts": 1514656762973.042,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762976.434, "dur": 1.779,
    "args": {
      "External id": 2484284,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656762977.120, "dur": 0.953,
    "args": {
      "External id": 2484285,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763023.184, "dur": 139.163,
    "args": {
      "External id": 2484286,"Record function id": 0, "Sequence number": 24740534, "Fwd thread id": 1, "Ev Idx": 1085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763025.216, "dur": 128.380,
    "args": {
      "External id": 2484287,"Sequence number": 24740534, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1086
    }
  },
  {
    "ph": "f", "id": 114, "pid": 1336760, "tid": 1381209, "ts": 1514656763025.216,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763030.677, "dur": 8.025,
    "args": {
      "External id": 2484288,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763032.342, "dur": 5.487,
    "args": {
      "External id": 2484289,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763036.253, "dur": 1.231,
    "args": {
      "External id": 2484290,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763040.088, "dur": 57.444,
    "args": {
      "External id": 2484291,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763098.645, "dur": 7.238,
    "args": {
      "External id": 2484292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763099.352, "dur": 5.919,
    "args": {
      "External id": 2484293,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763102.203, "dur": 2.938,
    "args": {
      "External id": 2484294,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763107.765, "dur": 5.656,
    "args": {
      "External id": 2484295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763109.599, "dur": 3.158,
    "args": {
      "External id": 2484296,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763112.229, "dur": 0.463,
    "args": {
      "External id": 2484297,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763114.127, "dur": 38.635,
    "args": {
      "External id": 2484298,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763169.243, "dur": 11.077,
    "args": {
      "External id": 2484299,"Record function id": 0, "Sequence number": 24740533, "Fwd thread id": 1, "Ev Idx": 1098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763170.316, "dur": 7.488,
    "args": {
      "External id": 2484300,"Sequence number": 24740533, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1099
    }
  },
  {
    "ph": "f", "id": 115, "pid": 1336760, "tid": 1381209, "ts": 1514656763170.316,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763171.962, "dur": 5.672,
    "args": {
      "External id": 2484301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763174.422, "dur": 3.088,
    "args": {
      "External id": 2484302,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763185.684, "dur": 7.975,
    "args": {
      "External id": 2484303,"Record function id": 0, "Sequence number": 24740532, "Fwd thread id": 1, "Ev Idx": 1102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763186.692, "dur": 5.229,
    "args": {
      "External id": 2484304,"Sequence number": 24740532, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1103
    }
  },
  {
    "ph": "f", "id": 116, "pid": 1336760, "tid": 1381209, "ts": 1514656763186.692,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763187.565, "dur": 4.139,
    "args": {
      "External id": 2484305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763188.218, "dur": 2.994,
    "args": {
      "External id": 2484306,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763190.835, "dur": 0.267,
    "args": {
      "External id": 2484307,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763197.885, "dur": 5.769,
    "args": {
      "External id": 2484308,"Record function id": 0, "Ev Idx": 1107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763199.398, "dur": 3.590,
    "args": {
      "External id": 2484309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763200.627, "dur": 2.014,
    "args": {
      "External id": 2484310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763201.292, "dur": 1.229,
    "args": {
      "External id": 2484311,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763208.367, "dur": 6.646,
    "args": {
      "External id": 2484312,"Record function id": 0, "Sequence number": 24740531, "Fwd thread id": 1, "Ev Idx": 1111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763209.167, "dur": 3.061,
    "args": {
      "External id": 2484313,"Sequence number": 24740531, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1112
    }
  },
  {
    "ph": "f", "id": 117, "pid": 1336760, "tid": 1381209, "ts": 1514656763209.167,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763210.314, "dur": 1.756,
    "args": {
      "External id": 2484314,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763210.756, "dur": 1.184,
    "args": {
      "External id": 2484315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763220.875, "dur": 295.328,
    "args": {
      "External id": 2484316,"Record function id": 0, "Sequence number": 24740530, "Fwd thread id": 1, "Ev Idx": 1115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763222.192, "dur": 272.727,
    "args": {
      "External id": 2484317,"Sequence number": 24740530, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1116
    }
  },
  {
    "ph": "f", "id": 118, "pid": 1336760, "tid": 1381209, "ts": 1514656763222.192,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763241.010, "dur": 6.270,
    "args": {
      "External id": 2484318,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763243.284, "dur": 3.613,
    "args": {
      "External id": 2484319,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763249.310, "dur": 6.435,
    "args": {
      "External id": 2484320,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763252.115, "dur": 3.383,
    "args": {
      "External id": 2484321,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763263.708, "dur": 5.211,
    "args": {
      "External id": 2484322,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763266.884, "dur": 1.847,
    "args": {
      "External id": 2484323,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763296.617, "dur": 173.019,
    "args": {
      "External id": 2484324,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763375.579, "dur": 4.780,
    "args": {
      "External id": 2484325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763382.044, "dur": 4.620,
    "args": {
      "External id": 2484326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763482.716, "dur": 3.214,
    "args": {
      "External id": 2484327,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763489.122, "dur": 0.697,
    "args": {
      "External id": 2484328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763491.621, "dur": 0.629,
    "args": {
      "External id": 2484329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763523.167, "dur": 259.618,
    "args": {
      "External id": 2484330,"Record function id": 0, "Sequence number": 24740529, "Fwd thread id": 1, "Ev Idx": 1129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763524.530, "dur": 250.071,
    "args": {
      "External id": 2484331,"Sequence number": 24740529, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1130
    }
  },
  {
    "ph": "f", "id": 119, "pid": 1336760, "tid": 1381209, "ts": 1514656763524.530,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763566.282, "dur": 49.292,
    "args": {
      "External id": 2484332,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763569.168, "dur": 4.522,
    "args": {
      "External id": 2484333,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763575.009, "dur": 39.817,
    "args": {
      "External id": 2484334,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763625.378, "dur": 6.650,
    "args": {
      "External id": 2484335,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763629.152, "dur": 2.585,
    "args": {
      "External id": 2484336,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763791.919, "dur": 160.187,
    "args": {
      "External id": 2484337,"Record function id": 0, "Sequence number": 24740528, "Fwd thread id": 1, "Ev Idx": 1136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763793.657, "dur": 151.359,
    "args": {
      "External id": 2484338,"Sequence number": 24740528, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1137
    }
  },
  {
    "ph": "f", "id": 120, "pid": 1336760, "tid": 1381209, "ts": 1514656763793.657,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763806.111, "dur": 44.138,
    "args": {
      "External id": 2484339,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763808.332, "dur": 2.919,
    "args": {
      "External id": 2484340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763817.019, "dur": 32.732,
    "args": {
      "External id": 2484341,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763857.097, "dur": 5.915,
    "args": {
      "External id": 2484342,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763860.408, "dur": 2.318,
    "args": {
      "External id": 2484343,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763958.374, "dur": 18.452,
    "args": {
      "External id": 2484344,"Record function id": 0, "Sequence number": 24740527, "Fwd thread id": 1, "Ev Idx": 1143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763960.256, "dur": 13.873,
    "args": {
      "External id": 2484345,"Sequence number": 24740527, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1144
    }
  },
  {
    "ph": "f", "id": 121, "pid": 1336760, "tid": 1381209, "ts": 1514656763960.256,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763962.290, "dur": 11.552,
    "args": {
      "External id": 2484346,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763965.803, "dur": 7.851,
    "args": {
      "External id": 2484347,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763980.394, "dur": 50.106,
    "args": {
      "External id": 2484348,"Record function id": 0, "Sequence number": 24740526, "Fwd thread id": 1, "Ev Idx": 1147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763981.620, "dur": 44.965,
    "args": {
      "External id": 2484349,"Sequence number": 24740526, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1148
    }
  },
  {
    "ph": "f", "id": 122, "pid": 1336760, "tid": 1381209, "ts": 1514656763981.620,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656763982.852, "dur": 43.525,
    "args": {
      "External id": 2484350,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764024.342, "dur": 1.698,
    "args": {
      "External id": 2484351,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764036.007, "dur": 7.127,
    "args": {
      "External id": 2484352,"Record function id": 0, "Sequence number": 24740525, "Fwd thread id": 1, "Ev Idx": 1151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764037.258, "dur": 3.187,
    "args": {
      "External id": 2484353,"Sequence number": 24740525, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1152
    }
  },
  {
    "ph": "f", "id": 123, "pid": 1336760, "tid": 1381209, "ts": 1514656764037.258,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764038.629, "dur": 1.677,
    "args": {
      "External id": 2484354,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764039.369, "dur": 0.803,
    "args": {
      "External id": 2484355,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764046.644, "dur": 10.944,
    "args": {
      "External id": 2484356,"Record function id": 0, "Sequence number": 24740524, "Fwd thread id": 1, "Ev Idx": 1155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764047.876, "dur": 7.270,
    "args": {
      "External id": 2484357,"Sequence number": 24740524, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1156
    }
  },
  {
    "ph": "f", "id": 124, "pid": 1336760, "tid": 1381209, "ts": 1514656764047.876,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764048.990, "dur": 5.998,
    "args": {
      "External id": 2484358,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764053.278, "dur": 1.577,
    "args": {
      "External id": 2484359,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764061.219, "dur": 160.344,
    "args": {
      "External id": 2484360,"Record function id": 0, "Sequence number": 24740523, "Fwd thread id": 1, "Ev Idx": 1159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764062.246, "dur": 151.205,
    "args": {
      "External id": 2484361,"Sequence number": 24740523, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1160
    }
  },
  {
    "ph": "f", "id": 125, "pid": 1336760, "tid": 1381209, "ts": 1514656764062.246,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764066.758, "dur": 7.306,
    "args": {
      "External id": 2484362,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764068.955, "dur": 4.430,
    "args": {
      "External id": 2484363,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764071.001, "dur": 2.057,
    "args": {
      "External id": 2484364,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764077.650, "dur": 72.268,
    "args": {
      "External id": 2484365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764151.185, "dur": 4.054,
    "args": {
      "External id": 2484366,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764151.993, "dur": 2.637,
    "args": {
      "External id": 2484367,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764153.337, "dur": 1.121,
    "args": {
      "External id": 2484368,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764157.025, "dur": 6.005,
    "args": {
      "External id": 2484369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764158.793, "dur": 3.374,
    "args": {
      "External id": 2484370,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764161.606, "dur": 0.473,
    "args": {
      "External id": 2484371,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764165.632, "dur": 46.565,
    "args": {
      "External id": 2484372,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764228.694, "dur": 7.162,
    "args": {
      "External id": 2484373,"Record function id": 0, "Sequence number": 24740522, "Fwd thread id": 1, "Ev Idx": 1172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764229.585, "dur": 3.940,
    "args": {
      "External id": 2484374,"Sequence number": 24740522, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1173
    }
  },
  {
    "ph": "f", "id": 126, "pid": 1336760, "tid": 1381209, "ts": 1514656764229.585,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764231.372, "dur": 2.021,
    "args": {
      "External id": 2484375,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764232.174, "dur": 1.080,
    "args": {
      "External id": 2484376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764239.292, "dur": 9.491,
    "args": {
      "External id": 2484377,"Record function id": 0, "Sequence number": 24740521, "Fwd thread id": 1, "Ev Idx": 1176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764240.444, "dur": 6.543,
    "args": {
      "External id": 2484378,"Sequence number": 24740521, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1177
    }
  },
  {
    "ph": "f", "id": 127, "pid": 1336760, "tid": 1381209, "ts": 1514656764240.444,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764241.591, "dur": 5.174,
    "args": {
      "External id": 2484379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764242.283, "dur": 3.993,
    "args": {
      "External id": 2484380,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764245.609, "dur": 0.583,
    "args": {
      "External id": 2484381,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764256.897, "dur": 9.152,
    "args": {
      "External id": 2484382,"Record function id": 0, "Ev Idx": 1181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764258.445, "dur": 6.851,
    "args": {
      "External id": 2484383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764261.004, "dur": 3.886,
    "args": {
      "External id": 2484384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764262.049, "dur": 2.752,
    "args": {
      "External id": 2484385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764269.433, "dur": 6.833,
    "args": {
      "External id": 2484386,"Record function id": 0, "Sequence number": 24740520, "Fwd thread id": 1, "Ev Idx": 1185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764270.505, "dur": 3.107,
    "args": {
      "External id": 2484387,"Sequence number": 24740520, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1186
    }
  },
  {
    "ph": "f", "id": 128, "pid": 1336760, "tid": 1381209, "ts": 1514656764270.505,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764272.026, "dur": 1.445,
    "args": {
      "External id": 2484388,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764272.563, "dur": 0.815,
    "args": {
      "External id": 2484389,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764279.522, "dur": 97.851,
    "args": {
      "External id": 2484390,"Record function id": 0, "Sequence number": 24740519, "Fwd thread id": 1, "Ev Idx": 1189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764280.434, "dur": 90.217,
    "args": {
      "External id": 2484391,"Sequence number": 24740519, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1190
    }
  },
  {
    "ph": "f", "id": 129, "pid": 1336760, "tid": 1381209, "ts": 1514656764280.434,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764284.676, "dur": 2.248,
    "args": {
      "External id": 2484392,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764285.248, "dur": 1.271,
    "args": {
      "External id": 2484393,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764285.843, "dur": 0.566,
    "args": {
      "External id": 2484394,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764287.502, "dur": 30.336,
    "args": {
      "External id": 2484395,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764318.945, "dur": 7.187,
    "args": {
      "External id": 2484396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764319.659, "dur": 5.941,
    "args": {
      "External id": 2484397,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764324.757, "dur": 0.736,
    "args": {
      "External id": 2484398,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764327.216, "dur": 5.606,
    "args": {
      "External id": 2484399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764329.101, "dur": 3.328,
    "args": {
      "External id": 2484400,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764329.819, "dur": 2.492,
    "args": {
      "External id": 2484401,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764333.492, "dur": 36.285,
    "args": {
      "External id": 2484402,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764381.961, "dur": 36.034,
    "args": {
      "External id": 2484403,"Record function id": 0, "Sequence number": 24740518, "Fwd thread id": 1, "Ev Idx": 1202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764382.912, "dur": 7.699,
    "args": {
      "External id": 2484404,"Sequence number": 24740518, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1203
    }
  },
  {
    "ph": "f", "id": 130, "pid": 1336760, "tid": 1381209, "ts": 1514656764382.912,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764386.507, "dur": 3.974,
    "args": {
      "External id": 2484405,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764389.336, "dur": 1.020,
    "args": {
      "External id": 2484406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764393.656, "dur": 21.870,
    "args": {
      "External id": 2484407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764424.189, "dur": 8.025,
    "args": {
      "External id": 2484408,"Record function id": 0, "Sequence number": 24740517, "Fwd thread id": 1, "Ev Idx": 1207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764425.503, "dur": 3.591,
    "args": {
      "External id": 2484409,"Sequence number": 24740517, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1208
    }
  },
  {
    "ph": "f", "id": 131, "pid": 1336760, "tid": 1381209, "ts": 1514656764425.503,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764426.473, "dur": 2.434,
    "args": {
      "External id": 2484410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764427.027, "dur": 1.348,
    "args": {
      "External id": 2484411,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764427.834, "dur": 0.429,
    "args": {
      "External id": 2484412,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764436.065, "dur": 7.300,
    "args": {
      "External id": 2484413,"Record function id": 0, "Ev Idx": 1212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764437.252, "dur": 5.632,
    "args": {
      "External id": 2484414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764438.349, "dur": 4.163,
    "args": {
      "External id": 2484415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764441.075, "dur": 1.342,
    "args": {
      "External id": 2484416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764446.654, "dur": 10.872,
    "args": {
      "External id": 2484417,"Record function id": 0, "Sequence number": 24740516, "Fwd thread id": 1, "Ev Idx": 1216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764447.620, "dur": 7.602,
    "args": {
      "External id": 2484418,"Sequence number": 24740516, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1217
    }
  },
  {
    "ph": "f", "id": 132, "pid": 1336760, "tid": 1381209, "ts": 1514656764447.620,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764448.739, "dur": 6.337,
    "args": {
      "External id": 2484419,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764454.180, "dur": 0.793,
    "args": {
      "External id": 2484420,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764460.682, "dur": 112.523,
    "args": {
      "External id": 2484421,"Record function id": 0, "Sequence number": 24740515, "Fwd thread id": 1, "Ev Idx": 1220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764461.359, "dur": 101.573,
    "args": {
      "External id": 2484422,"Sequence number": 24740515, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1221
    }
  },
  {
    "ph": "f", "id": 133, "pid": 1336760, "tid": 1381209, "ts": 1514656764461.359,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764465.657, "dur": 2.245,
    "args": {
      "External id": 2484423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764466.217, "dur": 1.280,
    "args": {
      "External id": 2484424,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764467.034, "dur": 0.357,
    "args": {
      "External id": 2484425,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764468.564, "dur": 28.360,
    "args": {
      "External id": 2484426,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764497.942, "dur": 6.428,
    "args": {
      "External id": 2484427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764498.404, "dur": 5.406,
    "args": {
      "External id": 2484428,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764501.364, "dur": 2.296,
    "args": {
      "External id": 2484429,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764505.341, "dur": 4.860,
    "args": {
      "External id": 2484430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764506.990, "dur": 2.826,
    "args": {
      "External id": 2484431,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764509.449, "dur": 0.285,
    "args": {
      "External id": 2484432,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764510.655, "dur": 50.845,
    "args": {
      "External id": 2484433,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764580.067, "dur": 26.938,
    "args": {
      "External id": 2484434,"Record function id": 0, "Sequence number": 24740514, "Fwd thread id": 1, "Ev Idx": 1233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764581.210, "dur": 4.463,
    "args": {
      "External id": 2484435,"Sequence number": 24740514, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1234
    }
  },
  {
    "ph": "f", "id": 134, "pid": 1336760, "tid": 1381209, "ts": 1514656764581.210,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764583.207, "dur": 2.324,
    "args": {
      "External id": 2484436,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764584.022, "dur": 1.375,
    "args": {
      "External id": 2484437,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764588.512, "dur": 16.383,
    "args": {
      "External id": 2484438,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764612.269, "dur": 7.510,
    "args": {
      "External id": 2484439,"Record function id": 0, "Sequence number": 24740513, "Fwd thread id": 1, "Ev Idx": 1238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764613.180, "dur": 4.611,
    "args": {
      "External id": 2484440,"Sequence number": 24740513, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1239
    }
  },
  {
    "ph": "f", "id": 135, "pid": 1336760, "tid": 1381209, "ts": 1514656764613.180,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764614.552, "dur": 3.052,
    "args": {
      "External id": 2484441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764615.231, "dur": 1.856,
    "args": {
      "External id": 2484442,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764616.234, "dur": 0.742,
    "args": {
      "External id": 2484443,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764626.098, "dur": 5.319,
    "args": {
      "External id": 2484444,"Record function id": 0, "Ev Idx": 1243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764627.323, "dur": 3.596,
    "args": {
      "External id": 2484445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764628.642, "dur": 2.025,
    "args": {
      "External id": 2484446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764629.232, "dur": 1.330,
    "args": {
      "External id": 2484447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764637.410, "dur": 336.265,
    "args": {
      "External id": 2484448,"Record function id": 0, "Sequence number": 24740512, "Fwd thread id": 1, "Ev Idx": 1247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764638.521, "dur": 306.945,
    "args": {
      "External id": 2484449,"Sequence number": 24740512, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1248
    }
  },
  {
    "ph": "f", "id": 136, "pid": 1336760, "tid": 1381209, "ts": 1514656764638.521,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764673.275, "dur": 1.919,
    "args": {
      "External id": 2484450,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764673.951, "dur": 1.103,
    "args": {
      "External id": 2484451,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764689.216, "dur": 6.184,
    "args": {
      "External id": 2484452,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764704.343, "dur": 1.779,
    "args": {
      "External id": 2484453,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764846.447, "dur": 1.326,
    "args": {
      "External id": 2484454,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764852.223, "dur": 31.527,
    "args": {
      "External id": 2484455,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764860.487, "dur": 0.919,
    "args": {
      "External id": 2484456,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764889.346, "dur": 33.272,
    "args": {
      "External id": 2484457,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764893.078, "dur": 29.345,
    "args": {
      "External id": 2484458,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764898.279, "dur": 6.265,
    "args": {
      "External id": 2484459,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764906.368, "dur": 15.453,
    "args": {
      "External id": 2484460,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764927.124, "dur": 2.339,
    "args": {
      "External id": 2484461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764928.248, "dur": 1.084,
    "args": {
      "External id": 2484462,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764935.051, "dur": 3.982,
    "args": {
      "External id": 2484463,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764937.787, "dur": 1.161,
    "args": {
      "External id": 2484464,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764954.838, "dur": 14.529,
    "args": {
      "External id": 2484465,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764981.692, "dur": 53.977,
    "args": {
      "External id": 2484466,"Record function id": 0, "Ev Idx": 1265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656764983.648, "dur": 50.519,
    "args": {
      "External id": 2484467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765028.569, "dur": 4.162,
    "args": {
      "External id": 2484468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765029.919, "dur": 2.549,
    "args": {
      "External id": 2484469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765041.387, "dur": 7.596,
    "args": {
      "External id": 2484470,"Record function id": 0, "Sequence number": 24740511, "Fwd thread id": 1, "Ev Idx": 1269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765043.334, "dur": 1.355,
    "args": {
      "External id": 2484471,"Sequence number": 24740511, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1270
    }
  },
  {
    "ph": "f", "id": 137, "pid": 1336760, "tid": 1381209, "ts": 1514656765043.334,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765053.264, "dur": 419.643,
    "args": {
      "External id": 2484472,"Record function id": 0, "Sequence number": 24740510, "Fwd thread id": 1, "Ev Idx": 1271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765056.104, "dur": 403.921,
    "args": {
      "External id": 2484473,"Sequence number": 24740510, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1272
    }
  },
  {
    "ph": "f", "id": 138, "pid": 1336760, "tid": 1381209, "ts": 1514656765056.104,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765092.147, "dur": 8.514,
    "args": {
      "External id": 2484474,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765096.959, "dur": 3.444,
    "args": {
      "External id": 2484475,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 1274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765104.344, "dur": 7.444,
    "args": {
      "External id": 2484476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765106.073, "dur": 4.886,
    "args": {
      "External id": 2484477,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765107.791, "dur": 2.966,
    "args": {
      "External id": 2484478,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765115.553, "dur": 91.677,
    "args": {
      "External id": 2484479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 1278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765118.561, "dur": 4.874,
    "args": {
      "External id": 2484480,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 1279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765119.277, "dur": 3.696,
    "args": {
      "External id": 2484481,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765122.431, "dur": 0.425,
    "args": {
      "External id": 2484482,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765124.842, "dur": 81.887,
    "args": {
      "External id": 2484483,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765126.588, "dur": 79.415,
    "args": {
      "External id": 2484484,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765211.167, "dur": 2.621,
    "args": {
      "External id": 2484485,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 1284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765212.442, "dur": 1.202,
    "args": {
      "External id": 2484486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765246.100, "dur": 6.221,
    "args": {
      "External id": 2484487,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765255.926, "dur": 2.076,
    "args": {
      "External id": 2484488,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765258.639, "dur": 2.246,
    "args": {
      "External id": 2484489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765291.292, "dur": 1.939,
    "args": {
      "External id": 2484490,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765291.963, "dur": 1.133,
    "args": {
      "External id": 2484491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765316.004, "dur": 123.753,
    "args": {
      "External id": 2484492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 1291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765321.260, "dur": 10.978,
    "args": {
      "External id": 2484493,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765326.244, "dur": 4.862,
    "args": {
      "External id": 2484494,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765333.476, "dur": 6.048,
    "args": {
      "External id": 2484495,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 1294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765338.218, "dur": 0.409,
    "args": {
      "External id": 2484496,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 1295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765340.832, "dur": 1.864,
    "args": {
      "External id": 2484497,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765341.869, "dur": 0.483,
    "args": {
      "External id": 2484498,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765345.156, "dur": 2.010,
    "args": {
      "External id": 2484499,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765345.829, "dur": 0.703,
    "args": {
      "External id": 2484500,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 1299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765350.898, "dur": 4.198,
    "args": {
      "External id": 2484501,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 1300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765354.074, "dur": 0.724,
    "args": {
      "External id": 2484502,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 1301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765355.923, "dur": 6.766,
    "args": {
      "External id": 2484503,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 1302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765360.729, "dur": 1.773,
    "args": {
      "External id": 2484504,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 1303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765365.125, "dur": 3.900,
    "args": {
      "External id": 2484505,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 1304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765368.004, "dur": 0.710,
    "args": {
      "External id": 2484506,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 1305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765369.549, "dur": 3.711,
    "args": {
      "External id": 2484507,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765370.213, "dur": 2.965,
    "args": {
      "External id": 2484508,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765374.401, "dur": 51.206,
    "args": {
      "External id": 2484509,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 1308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765429.404, "dur": 1.159,
    "args": {
      "External id": 2484510,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 1309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765431.173, "dur": 2.227,
    "args": {
      "External id": 2484511,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 1310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765432.273, "dur": 0.456,
    "args": {
      "External id": 2484512,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 1311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765437.596, "dur": 1.035,
    "args": {
      "External id": 2484513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 1312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765482.266, "dur": 7.718,
    "args": {
      "External id": 2484514,"Record function id": 0, "Ev Idx": 1313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765484.130, "dur": 5.249,
    "args": {
      "External id": 2484515,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765485.993, "dur": 2.668,
    "args": {
      "External id": 2484516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765486.874, "dur": 1.707,
    "args": {
      "External id": 2484517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765493.990, "dur": 6.913,
    "args": {
      "External id": 2484518,"Record function id": 0, "Sequence number": 24740509, "Fwd thread id": 1, "Ev Idx": 1317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765494.933, "dur": 3.774,
    "args": {
      "External id": 2484519,"Sequence number": 24740509, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1318
    }
  },
  {
    "ph": "f", "id": 139, "pid": 1336760, "tid": 1381209, "ts": 1514656765494.933,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765496.429, "dur": 2.121,
    "args": {
      "External id": 2484520,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765497.364, "dur": 1.058,
    "args": {
      "External id": 2484521,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765504.543, "dur": 153.493,
    "args": {
      "External id": 2484522,"Record function id": 0, "Sequence number": 24740508, "Fwd thread id": 1, "Ev Idx": 1321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765505.485, "dur": 143.994,
    "args": {
      "External id": 2484523,"Sequence number": 24740508, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1322
    }
  },
  {
    "ph": "f", "id": 140, "pid": 1336760, "tid": 1381209, "ts": 1514656765505.485,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765508.886, "dur": 5.461,
    "args": {
      "External id": 2484524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765509.930, "dur": 3.947,
    "args": {
      "External id": 2484525,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765513.017, "dur": 0.724,
    "args": {
      "External id": 2484526,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765515.755, "dur": 65.291,
    "args": {
      "External id": 2484527,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765583.353, "dur": 6.727,
    "args": {
      "External id": 2484528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765584.120, "dur": 4.845,
    "args": {
      "External id": 2484529,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765587.301, "dur": 1.485,
    "args": {
      "External id": 2484530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765591.879, "dur": 7.035,
    "args": {
      "External id": 2484531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765595.481, "dur": 3.002,
    "args": {
      "External id": 2484532,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765596.425, "dur": 1.954,
    "args": {
      "External id": 2484533,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765599.610, "dur": 49.138,
    "args": {
      "External id": 2484534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765664.514, "dur": 6.306,
    "args": {
      "External id": 2484535,"Record function id": 0, "Sequence number": 24740507, "Fwd thread id": 1, "Ev Idx": 1334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765665.636, "dur": 3.830,
    "args": {
      "External id": 2484536,"Sequence number": 24740507, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1335
    }
  },
  {
    "ph": "f", "id": 141, "pid": 1336760, "tid": 1381209, "ts": 1514656765665.636,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765667.065, "dur": 2.263,
    "args": {
      "External id": 2484537,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765667.792, "dur": 1.446,
    "args": {
      "External id": 2484538,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765674.074, "dur": 11.314,
    "args": {
      "External id": 2484539,"Record function id": 0, "Sequence number": 24740506, "Fwd thread id": 1, "Ev Idx": 1338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765675.473, "dur": 6.868,
    "args": {
      "External id": 2484540,"Sequence number": 24740506, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1339
    }
  },
  {
    "ph": "f", "id": 142, "pid": 1336760, "tid": 1381209, "ts": 1514656765675.473,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765678.095, "dur": 4.041,
    "args": {
      "External id": 2484541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765678.705, "dur": 3.004,
    "args": {
      "External id": 2484542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765681.019, "dur": 0.605,
    "args": {
      "External id": 2484543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765689.339, "dur": 5.379,
    "args": {
      "External id": 2484544,"Record function id": 0, "Ev Idx": 1343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765690.539, "dur": 3.721,
    "args": {
      "External id": 2484545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765691.816, "dur": 2.130,
    "args": {
      "External id": 2484546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765692.414, "dur": 1.447,
    "args": {
      "External id": 2484547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765697.947, "dur": 7.626,
    "args": {
      "External id": 2484548,"Record function id": 0, "Sequence number": 24740505, "Fwd thread id": 1, "Ev Idx": 1347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765698.965, "dur": 4.530,
    "args": {
      "External id": 2484549,"Sequence number": 24740505, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1348
    }
  },
  {
    "ph": "f", "id": 143, "pid": 1336760, "tid": 1381209, "ts": 1514656765698.965,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765700.116, "dur": 3.247,
    "args": {
      "External id": 2484550,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765702.378, "dur": 0.891,
    "args": {
      "External id": 2484551,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765708.979, "dur": 116.534,
    "args": {
      "External id": 2484552,"Record function id": 0, "Sequence number": 24740504, "Fwd thread id": 1, "Ev Idx": 1351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765709.943, "dur": 107.356,
    "args": {
      "External id": 2484553,"Sequence number": 24740504, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1352
    }
  },
  {
    "ph": "f", "id": 144, "pid": 1336760, "tid": 1381209, "ts": 1514656765709.943,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765713.036, "dur": 4.074,
    "args": {
      "External id": 2484554,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765713.630, "dur": 3.094,
    "args": {
      "External id": 2484555,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765716.140, "dur": 0.489,
    "args": {
      "External id": 2484556,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765717.788, "dur": 32.406,
    "args": {
      "External id": 2484557,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765753.122, "dur": 6.077,
    "args": {
      "External id": 2484558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765754.157, "dur": 4.394,
    "args": {
      "External id": 2484559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765755.004, "dur": 3.355,
    "args": {
      "External id": 2484560,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765760.307, "dur": 3.479,
    "args": {
      "External id": 2484561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765762.153, "dur": 1.261,
    "args": {
      "External id": 2484562,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765762.807, "dur": 0.538,
    "args": {
      "External id": 2484563,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765766.034, "dur": 50.639,
    "args": {
      "External id": 2484564,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765832.325, "dur": 32.102,
    "args": {
      "External id": 2484565,"Record function id": 0, "Sequence number": 24740503, "Fwd thread id": 1, "Ev Idx": 1364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765833.301, "dur": 5.173,
    "args": {
      "External id": 2484566,"Sequence number": 24740503, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1365
    }
  },
  {
    "ph": "f", "id": 145, "pid": 1336760, "tid": 1381209, "ts": 1514656765833.301,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765836.274, "dur": 2.045,
    "args": {
      "External id": 2484567,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765837.140, "dur": 1.087,
    "args": {
      "External id": 2484568,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765841.301, "dur": 19.943,
    "args": {
      "External id": 2484569,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765868.237, "dur": 9.873,
    "args": {
      "External id": 2484570,"Record function id": 0, "Sequence number": 24740502, "Fwd thread id": 1, "Ev Idx": 1369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765869.357, "dur": 6.104,
    "args": {
      "External id": 2484571,"Sequence number": 24740502, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1370
    }
  },
  {
    "ph": "f", "id": 146, "pid": 1336760, "tid": 1381209, "ts": 1514656765869.357,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765870.386, "dur": 4.851,
    "args": {
      "External id": 2484572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765871.306, "dur": 3.411,
    "args": {
      "External id": 2484573,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765874.062, "dur": 0.535,
    "args": {
      "External id": 2484574,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765881.930, "dur": 6.740,
    "args": {
      "External id": 2484575,"Record function id": 0, "Ev Idx": 1374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765885.023, "dur": 3.185,
    "args": {
      "External id": 2484576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765885.964, "dur": 1.833,
    "args": {
      "External id": 2484577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765886.432, "dur": 1.281,
    "args": {
      "External id": 2484578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765892.700, "dur": 474.063,
    "args": {
      "External id": 2484579,"Record function id": 0, "Sequence number": 24740501, "Fwd thread id": 1, "Ev Idx": 1378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765897.874, "dur": 432.259,
    "args": {
      "External id": 2484580,"Sequence number": 24740501, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1379
    }
  },
  {
    "ph": "f", "id": 147, "pid": 1336760, "tid": 1381209, "ts": 1514656765897.874,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765920.072, "dur": 32.679,
    "args": {
      "External id": 2484581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765921.667, "dur": 30.882,
    "args": {
      "External id": 2484582,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765924.337, "dur": 5.996,
    "args": {
      "External id": 2484583,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765926.754, "dur": 3.049,
    "args": {
      "External id": 2484584,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765931.586, "dur": 20.404,
    "args": {
      "External id": 2484585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765966.585, "dur": 6.077,
    "args": {
      "External id": 2484586,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765969.383, "dur": 3.144,
    "args": {
      "External id": 2484587,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765976.292, "dur": 1.542,
    "args": {
      "External id": 2484588,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656765976.877, "dur": 0.861,
    "args": {
      "External id": 2484589,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766031.585, "dur": 3.917,
    "args": {
      "External id": 2484590,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766047.254, "dur": 4.289,
    "args": {
      "External id": 2484591,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766211.081, "dur": 4.594,
    "args": {
      "External id": 2484592,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766219.619, "dur": 33.987,
    "args": {
      "External id": 2484593,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766230.601, "dur": 0.934,
    "args": {
      "External id": 2484594,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766259.534, "dur": 33.856,
    "args": {
      "External id": 2484595,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766261.268, "dur": 31.941,
    "args": {
      "External id": 2484596,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766264.674, "dur": 4.047,
    "args": {
      "External id": 2484597,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766272.372, "dur": 20.113,
    "args": {
      "External id": 2484598,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766297.827, "dur": 3.974,
    "args": {
      "External id": 2484599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766300.639, "dur": 1.033,
    "args": {
      "External id": 2484600,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766308.803, "dur": 4.352,
    "args": {
      "External id": 2484601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766309.765, "dur": 3.247,
    "args": {
      "External id": 2484602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766314.991, "dur": 4.112,
    "args": {
      "External id": 2484603,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766317.875, "dur": 1.144,
    "args": {
      "External id": 2484604,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766347.188, "dur": 17.920,
    "args": {
      "External id": 2484605,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766379.767, "dur": 9.782,
    "args": {
      "External id": 2484606,"Record function id": 0, "Ev Idx": 1405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766382.227, "dur": 6.535,
    "args": {
      "External id": 2484607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766384.318, "dur": 3.453,
    "args": {
      "External id": 2484608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766385.490, "dur": 2.129,
    "args": {
      "External id": 2484609,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766393.214, "dur": 9.133,
    "args": {
      "External id": 2484610,"Record function id": 0, "Sequence number": 24740500, "Fwd thread id": 1, "Ev Idx": 1409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766394.376, "dur": 5.422,
    "args": {
      "External id": 2484611,"Sequence number": 24740500, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1410
    }
  },
  {
    "ph": "f", "id": 148, "pid": 1336760, "tid": 1381209, "ts": 1514656766394.376,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766395.803, "dur": 3.827,
    "args": {
      "External id": 2484612,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766398.407, "dur": 1.066,
    "args": {
      "External id": 2484613,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766406.124, "dur": 159.258,
    "args": {
      "External id": 2484614,"Record function id": 0, "Sequence number": 24740499, "Fwd thread id": 1, "Ev Idx": 1413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766406.958, "dur": 149.321,
    "args": {
      "External id": 2484615,"Sequence number": 24740499, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1414
    }
  },
  {
    "ph": "f", "id": 149, "pid": 1336760, "tid": 1381209, "ts": 1514656766406.958,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766410.410, "dur": 4.211,
    "args": {
      "External id": 2484616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766411.807, "dur": 2.175,
    "args": {
      "External id": 2484617,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766413.036, "dur": 0.727,
    "args": {
      "External id": 2484618,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766415.784, "dur": 60.208,
    "args": {
      "External id": 2484619,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766478.930, "dur": 7.605,
    "args": {
      "External id": 2484620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766480.085, "dur": 5.785,
    "args": {
      "External id": 2484621,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766483.118, "dur": 2.571,
    "args": {
      "External id": 2484622,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766488.201, "dur": 3.172,
    "args": {
      "External id": 2484623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766489.636, "dur": 1.164,
    "args": {
      "External id": 2484624,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766490.272, "dur": 0.461,
    "args": {
      "External id": 2484625,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766494.066, "dur": 60.841,
    "args": {
      "External id": 2484626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766572.808, "dur": 12.137,
    "args": {
      "External id": 2484627,"Record function id": 0, "Sequence number": 24740498, "Fwd thread id": 1, "Ev Idx": 1426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766574.075, "dur": 8.130,
    "args": {
      "External id": 2484628,"Sequence number": 24740498, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1427
    }
  },
  {
    "ph": "f", "id": 150, "pid": 1336760, "tid": 1381209, "ts": 1514656766574.075,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766575.853, "dur": 6.193,
    "args": {
      "External id": 2484629,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766578.922, "dur": 2.986,
    "args": {
      "External id": 2484630,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766588.441, "dur": 10.049,
    "args": {
      "External id": 2484631,"Record function id": 0, "Sequence number": 24740497, "Fwd thread id": 1, "Ev Idx": 1430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766589.561, "dur": 7.039,
    "args": {
      "External id": 2484632,"Sequence number": 24740497, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1431
    }
  },
  {
    "ph": "f", "id": 151, "pid": 1336760, "tid": 1381209, "ts": 1514656766589.561,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766590.394, "dur": 6.012,
    "args": {
      "External id": 2484633,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766591.467, "dur": 4.466,
    "args": {
      "External id": 2484634,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766594.984, "dur": 0.830,
    "args": {
      "External id": 2484635,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766602.666, "dur": 5.645,
    "args": {
      "External id": 2484636,"Record function id": 0, "Ev Idx": 1435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766604.304, "dur": 3.504,
    "args": {
      "External id": 2484637,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766605.621, "dur": 1.867,
    "args": {
      "External id": 2484638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766606.154, "dur": 1.227,
    "args": {
      "External id": 2484639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766611.408, "dur": 6.642,
    "args": {
      "External id": 2484640,"Record function id": 0, "Sequence number": 24740496, "Fwd thread id": 1, "Ev Idx": 1439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766612.256, "dur": 3.201,
    "args": {
      "External id": 2484641,"Sequence number": 24740496, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1440
    }
  },
  {
    "ph": "f", "id": 152, "pid": 1336760, "tid": 1381209, "ts": 1514656766612.256,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766613.584, "dur": 1.704,
    "args": {
      "External id": 2484642,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766614.039, "dur": 1.114,
    "args": {
      "External id": 2484643,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766622.583, "dur": 288.338,
    "args": {
      "External id": 2484644,"Record function id": 0, "Sequence number": 24740495, "Fwd thread id": 1, "Ev Idx": 1443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766625.433, "dur": 264.101,
    "args": {
      "External id": 2484645,"Sequence number": 24740495, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1444
    }
  },
  {
    "ph": "f", "id": 153, "pid": 1336760, "tid": 1381209, "ts": 1514656766625.433,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766641.443, "dur": 8.787,
    "args": {
      "External id": 2484646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766645.821, "dur": 4.013,
    "args": {
      "External id": 2484647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766652.190, "dur": 5.222,
    "args": {
      "External id": 2484648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766655.352, "dur": 1.884,
    "args": {
      "External id": 2484649,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766658.827, "dur": 2.506,
    "args": {
      "External id": 2484650,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766659.603, "dur": 1.567,
    "args": {
      "External id": 2484651,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766684.771, "dur": 180.198,
    "args": {
      "External id": 2484652,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766764.536, "dur": 4.946,
    "args": {
      "External id": 2484653,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766773.043, "dur": 4.518,
    "args": {
      "External id": 2484654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766876.890, "dur": 3.228,
    "args": {
      "External id": 2484655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766883.108, "dur": 0.748,
    "args": {
      "External id": 2484656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766886.067, "dur": 0.620,
    "args": {
      "External id": 2484657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766917.801, "dur": 279.048,
    "args": {
      "External id": 2484658,"Record function id": 0, "Sequence number": 24740494, "Fwd thread id": 1, "Ev Idx": 1457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766919.318, "dur": 269.179,
    "args": {
      "External id": 2484659,"Sequence number": 24740494, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1458
    }
  },
  {
    "ph": "f", "id": 154, "pid": 1336760, "tid": 1381209, "ts": 1514656766919.318,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766938.881, "dur": 97.250,
    "args": {
      "External id": 2484660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766943.446, "dur": 2.538,
    "args": {
      "External id": 2484661,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656766947.452, "dur": 87.422,
    "args": {
      "External id": 2484662,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767048.211, "dur": 5.512,
    "args": {
      "External id": 2484663,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767049.877, "dur": 3.532,
    "args": {
      "External id": 2484664,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767205.593, "dur": 153.710,
    "args": {
      "External id": 2484665,"Record function id": 0, "Sequence number": 24740493, "Fwd thread id": 1, "Ev Idx": 1464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767209.308, "dur": 142.363,
    "args": {
      "External id": 2484666,"Sequence number": 24740493, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1465
    }
  },
  {
    "ph": "f", "id": 155, "pid": 1336760, "tid": 1381209, "ts": 1514656767209.308,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767222.139, "dur": 34.308,
    "args": {
      "External id": 2484667,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767224.326, "dur": 2.420,
    "args": {
      "External id": 2484668,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767227.749, "dur": 28.225,
    "args": {
      "External id": 2484669,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767262.979, "dur": 3.846,
    "args": {
      "External id": 2484670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767264.085, "dur": 2.500,
    "args": {
      "External id": 2484671,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767365.358, "dur": 16.518,
    "args": {
      "External id": 2484672,"Record function id": 0, "Sequence number": 24740492, "Fwd thread id": 1, "Ev Idx": 1471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767366.870, "dur": 12.462,
    "args": {
      "External id": 2484673,"Sequence number": 24740492, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1472
    }
  },
  {
    "ph": "f", "id": 156, "pid": 1336760, "tid": 1381209, "ts": 1514656767366.870,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767371.276, "dur": 7.775,
    "args": {
      "External id": 2484674,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767372.496, "dur": 6.366,
    "args": {
      "External id": 2484675,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767385.618, "dur": 6.916,
    "args": {
      "External id": 2484676,"Record function id": 0, "Sequence number": 24740491, "Fwd thread id": 1, "Ev Idx": 1475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767386.917, "dur": 3.398,
    "args": {
      "External id": 2484677,"Sequence number": 24740491, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1476
    }
  },
  {
    "ph": "f", "id": 157, "pid": 1336760, "tid": 1381209, "ts": 1514656767386.917,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767388.148, "dur": 2.025,
    "args": {
      "External id": 2484678,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767389.091, "dur": 0.945,
    "args": {
      "External id": 2484679,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767395.846, "dur": 10.682,
    "args": {
      "External id": 2484680,"Record function id": 0, "Sequence number": 24740490, "Fwd thread id": 1, "Ev Idx": 1479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767396.687, "dur": 6.871,
    "args": {
      "External id": 2484681,"Sequence number": 24740490, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1480
    }
  },
  {
    "ph": "f", "id": 158, "pid": 1336760, "tid": 1381209, "ts": 1514656767396.687,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767401.680, "dur": 1.737,
    "args": {
      "External id": 2484682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767402.227, "dur": 1.071,
    "args": {
      "External id": 2484683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767409.843, "dur": 6.139,
    "args": {
      "External id": 2484684,"Record function id": 0, "Sequence number": 24740489, "Fwd thread id": 1, "Ev Idx": 1483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767410.935, "dur": 3.032,
    "args": {
      "External id": 2484685,"Sequence number": 24740489, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1484
    }
  },
  {
    "ph": "f", "id": 159, "pid": 1336760, "tid": 1381209, "ts": 1514656767410.935,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767412.243, "dur": 1.584,
    "args": {
      "External id": 2484686,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767412.952, "dur": 0.740,
    "args": {
      "External id": 2484687,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767419.606, "dur": 178.492,
    "args": {
      "External id": 2484688,"Record function id": 0, "Sequence number": 24740488, "Fwd thread id": 1, "Ev Idx": 1487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767420.653, "dur": 167.313,
    "args": {
      "External id": 2484689,"Sequence number": 24740488, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1488
    }
  },
  {
    "ph": "f", "id": 160, "pid": 1336760, "tid": 1381209, "ts": 1514656767420.653,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767425.213, "dur": 10.574,
    "args": {
      "External id": 2484690,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767428.801, "dur": 6.353,
    "args": {
      "External id": 2484691,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767432.987, "dur": 1.929,
    "args": {
      "External id": 2484692,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767437.502, "dur": 67.194,
    "args": {
      "External id": 2484693,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767506.030, "dur": 4.590,
    "args": {
      "External id": 2484694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767506.817, "dur": 2.943,
    "args": {
      "External id": 2484695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767508.041, "dur": 1.542,
    "args": {
      "External id": 2484696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767512.543, "dur": 5.670,
    "args": {
      "External id": 2484697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767516.336, "dur": 1.348,
    "args": {
      "External id": 2484698,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767516.996, "dur": 0.613,
    "args": {
      "External id": 2484699,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767518.898, "dur": 67.664,
    "args": {
      "External id": 2484700,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767605.154, "dur": 9.510,
    "args": {
      "External id": 2484701,"Record function id": 0, "Sequence number": 24740487, "Fwd thread id": 1, "Ev Idx": 1500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767606.418, "dur": 6.294,
    "args": {
      "External id": 2484702,"Sequence number": 24740487, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1501
    }
  },
  {
    "ph": "f", "id": 161, "pid": 1336760, "tid": 1381209, "ts": 1514656767606.418,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767609.950, "dur": 2.619,
    "args": {
      "External id": 2484703,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767610.642, "dur": 1.780,
    "args": {
      "External id": 2484704,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767617.944, "dur": 9.029,
    "args": {
      "External id": 2484705,"Record function id": 0, "Sequence number": 24740486, "Fwd thread id": 1, "Ev Idx": 1504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767618.984, "dur": 5.853,
    "args": {
      "External id": 2484706,"Sequence number": 24740486, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1505
    }
  },
  {
    "ph": "f", "id": 162, "pid": 1336760, "tid": 1381209, "ts": 1514656767618.984,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767621.686, "dur": 2.936,
    "args": {
      "External id": 2484707,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767622.511, "dur": 1.641,
    "args": {
      "External id": 2484708,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767623.573, "dur": 0.464,
    "args": {
      "External id": 2484709,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767632.907, "dur": 9.178,
    "args": {
      "External id": 2484710,"Record function id": 0, "Ev Idx": 1509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767634.547, "dur": 6.823,
    "args": {
      "External id": 2484711,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767637.114, "dur": 3.816,
    "args": {
      "External id": 2484712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767638.202, "dur": 2.633,
    "args": {
      "External id": 2484713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767645.728, "dur": 10.208,
    "args": {
      "External id": 2484714,"Record function id": 0, "Sequence number": 24740485, "Fwd thread id": 1, "Ev Idx": 1513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767648.976, "dur": 5.157,
    "args": {
      "External id": 2484715,"Sequence number": 24740485, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1514
    }
  },
  {
    "ph": "f", "id": 163, "pid": 1336760, "tid": 1381209, "ts": 1514656767648.976,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767650.531, "dur": 3.431,
    "args": {
      "External id": 2484716,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767653.142, "dur": 0.721,
    "args": {
      "External id": 2484717,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767659.003, "dur": 93.081,
    "args": {
      "External id": 2484718,"Record function id": 0, "Sequence number": 24740484, "Fwd thread id": 1, "Ev Idx": 1517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767659.866, "dur": 85.677,
    "args": {
      "External id": 2484719,"Sequence number": 24740484, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1518
    }
  },
  {
    "ph": "f", "id": 164, "pid": 1336760, "tid": 1381209, "ts": 1514656767659.866,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767662.895, "dur": 2.400,
    "args": {
      "External id": 2484720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767663.490, "dur": 1.341,
    "args": {
      "External id": 2484721,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767664.199, "dur": 0.496,
    "args": {
      "External id": 2484722,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767666.277, "dur": 28.936,
    "args": {
      "External id": 2484723,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767698.259, "dur": 3.014,
    "args": {
      "External id": 2484724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767698.774, "dur": 1.940,
    "args": {
      "External id": 2484725,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767699.468, "dur": 1.112,
    "args": {
      "External id": 2484726,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767702.376, "dur": 6.443,
    "args": {
      "External id": 2484727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767703.594, "dur": 4.881,
    "args": {
      "External id": 2484728,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767705.971, "dur": 2.432,
    "args": {
      "External id": 2484729,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767711.395, "dur": 33.372,
    "args": {
      "External id": 2484730,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767758.263, "dur": 36.199,
    "args": {
      "External id": 2484731,"Record function id": 0, "Sequence number": 24740483, "Fwd thread id": 1, "Ev Idx": 1530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767759.908, "dur": 7.539,
    "args": {
      "External id": 2484732,"Sequence number": 24740483, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1531
    }
  },
  {
    "ph": "f", "id": 165, "pid": 1336760, "tid": 1381209, "ts": 1514656767759.908,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767761.454, "dur": 5.831,
    "args": {
      "External id": 2484733,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767765.898, "dur": 1.245,
    "args": {
      "External id": 2484734,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767770.738, "dur": 21.166,
    "args": {
      "External id": 2484735,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767798.345, "dur": 34.955,
    "args": {
      "External id": 2484736,"Record function id": 0, "Sequence number": 24740482, "Fwd thread id": 1, "Ev Idx": 1535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767823.842, "dur": 7.536,
    "args": {
      "External id": 2484737,"Sequence number": 24740482, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1536
    }
  },
  {
    "ph": "f", "id": 166, "pid": 1336760, "tid": 1381209, "ts": 1514656767823.842,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767826.707, "dur": 4.480,
    "args": {
      "External id": 2484738,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767827.463, "dur": 3.205,
    "args": {
      "External id": 2484739,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767830.147, "dur": 0.420,
    "args": {
      "External id": 2484740,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767837.042, "dur": 4.363,
    "args": {
      "External id": 2484741,"Record function id": 0, "Ev Idx": 1540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767838.136, "dur": 2.790,
    "args": {
      "External id": 2484742,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767839.107, "dur": 1.453,
    "args": {
      "External id": 2484743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767839.512, "dur": 0.969,
    "args": {
      "External id": 2484744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767844.594, "dur": 7.419,
    "args": {
      "External id": 2484745,"Record function id": 0, "Sequence number": 24740481, "Fwd thread id": 1, "Ev Idx": 1544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767845.689, "dur": 4.082,
    "args": {
      "External id": 2484746,"Sequence number": 24740481, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1545
    }
  },
  {
    "ph": "f", "id": 167, "pid": 1336760, "tid": 1381209, "ts": 1514656767845.689,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767846.568, "dur": 3.060,
    "args": {
      "External id": 2484747,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767848.680, "dur": 0.829,
    "args": {
      "External id": 2484748,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767855.171, "dur": 92.688,
    "args": {
      "External id": 2484749,"Record function id": 0, "Sequence number": 24740480, "Fwd thread id": 1, "Ev Idx": 1548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767855.863, "dur": 83.579,
    "args": {
      "External id": 2484750,"Sequence number": 24740480, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1549
    }
  },
  {
    "ph": "f", "id": 168, "pid": 1336760, "tid": 1381209, "ts": 1514656767855.863,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767859.044, "dur": 4.492,
    "args": {
      "External id": 2484751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767859.602, "dur": 3.499,
    "args": {
      "External id": 2484752,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767862.611, "dur": 0.404,
    "args": {
      "External id": 2484753,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767864.270, "dur": 28.002,
    "args": {
      "External id": 2484754,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767895.270, "dur": 2.340,
    "args": {
      "External id": 2484755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767895.727, "dur": 1.370,
    "args": {
      "External id": 2484756,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767896.449, "dur": 0.542,
    "args": {
      "External id": 2484757,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767898.698, "dur": 2.939,
    "args": {
      "External id": 2484758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767900.020, "dur": 1.143,
    "args": {
      "External id": 2484759,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767900.582, "dur": 0.512,
    "args": {
      "External id": 2484760,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767903.953, "dur": 34.775,
    "args": {
      "External id": 2484761,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767952.257, "dur": 25.629,
    "args": {
      "External id": 2484762,"Record function id": 0, "Sequence number": 24740479, "Fwd thread id": 1, "Ev Idx": 1561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767953.299, "dur": 5.343,
    "args": {
      "External id": 2484763,"Sequence number": 24740479, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1562
    }
  },
  {
    "ph": "f", "id": 169, "pid": 1336760, "tid": 1381209, "ts": 1514656767953.299,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767956.890, "dur": 1.592,
    "args": {
      "External id": 2484764,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767957.355, "dur": 1.020,
    "args": {
      "External id": 2484765,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767960.933, "dur": 14.506,
    "args": {
      "External id": 2484766,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767981.481, "dur": 48.673,
    "args": {
      "External id": 2484767,"Record function id": 0, "Sequence number": 24740478, "Fwd thread id": 1, "Ev Idx": 1566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767982.321, "dur": 44.807,
    "args": {
      "External id": 2484768,"Sequence number": 24740478, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1567
    }
  },
  {
    "ph": "f", "id": 170, "pid": 1336760, "tid": 1381209, "ts": 1514656767982.321,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767983.199, "dur": 43.688,
    "args": {
      "External id": 2484769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1381209,
    "ts": 1514656767983.933, "dur": 42.011,
    "args": {
      "External id": 2484770,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768024.794, "dur": 0.766,
    "args": {
      "External id": 2484771,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768036.172, "dur": 5.850,
    "args": {
      "External id": 2484772,"Record function id": 0, "Ev Idx": 1571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768037.792, "dur": 3.789,
    "args": {
      "External id": 2484773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768039.063, "dur": 2.054,
    "args": {
      "External id": 2484774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768039.662, "dur": 1.348,
    "args": {
      "External id": 2484775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768046.279, "dur": 352.651,
    "args": {
      "External id": 2484776,"Record function id": 0, "Sequence number": 24740477, "Fwd thread id": 1, "Ev Idx": 1575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768049.286, "dur": 313.585,
    "args": {
      "External id": 2484777,"Sequence number": 24740477, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1576
    }
  },
  {
    "ph": "f", "id": 171, "pid": 1336760, "tid": 1381209, "ts": 1514656768049.286,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768081.959, "dur": 2.093,
    "args": {
      "External id": 2484778,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768082.568, "dur": 1.341,
    "args": {
      "External id": 2484779,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768099.193, "dur": 6.486,
    "args": {
      "External id": 2484780,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768115.340, "dur": 2.300,
    "args": {
      "External id": 2484781,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768259.026, "dur": 1.753,
    "args": {
      "External id": 2484782,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768264.427, "dur": 39.223,
    "args": {
      "External id": 2484783,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768276.375, "dur": 0.991,
    "args": {
      "External id": 2484784,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768309.646, "dur": 30.721,
    "args": {
      "External id": 2484785,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768311.582, "dur": 28.552,
    "args": {
      "External id": 2484786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768315.349, "dur": 4.681,
    "args": {
      "External id": 2484787,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768323.299, "dur": 16.298,
    "args": {
      "External id": 2484788,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768344.692, "dur": 4.529,
    "args": {
      "External id": 2484789,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768348.182, "dur": 0.915,
    "args": {
      "External id": 2484790,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768354.615, "dur": 1.973,
    "args": {
      "External id": 2484791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768355.359, "dur": 1.142,
    "args": {
      "External id": 2484792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768378.203, "dur": 15.311,
    "args": {
      "External id": 2484793,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768407.918, "dur": 9.665,
    "args": {
      "External id": 2484794,"Record function id": 0, "Ev Idx": 1593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768409.721, "dur": 7.260,
    "args": {
      "External id": 2484795,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768411.258, "dur": 4.785,
    "args": {
      "External id": 2484796,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768414.357, "dur": 1.604,
    "args": {
      "External id": 2484797,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768421.791, "dur": 2802.061,
    "args": {
      "External id": 2484798,"Record function id": 0, "Ev Idx": 1597
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.27)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768454.673, "dur": 985.642,
    "args": {
      "External id": 2484799,"Record function id": 0, "Ev Idx": 1598
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.26", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768482.937, "dur": 948.582,
    "args": {
      "External id": 2484800,"Record function id": 0, "Ev Idx": 1599
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768495.929, "dur": 920.756,
    "args": {
      "External id": 2484801,"Record function id": 0, "Ev Idx": 1600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768617.519, "dur": 7.899,
    "args": {
      "External id": 2484802,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768643.697, "dur": 38.227,
    "args": {
      "External id": 2484803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768650.387, "dur": 3.496,
    "args": {
      "External id": 2484804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768655.003, "dur": 0.747,
    "args": {
      "External id": 2484805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768660.333, "dur": 0.587,
    "args": {
      "External id": 2484806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768661.595, "dur": 0.312,
    "args": {
      "External id": 2484807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768662.487, "dur": 0.548,
    "args": {
      "External id": 2484808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768667.459, "dur": 0.358,
    "args": {
      "External id": 2484809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768668.446, "dur": 0.429,
    "args": {
      "External id": 2484810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768669.452, "dur": 2.254,
    "args": {
      "External id": 2484811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768674.051, "dur": 1.918,
    "args": {
      "External id": 2484812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768697.363, "dur": 39.703,
    "args": {
      "External id": 2484813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768781.603, "dur": 117.380,
    "args": {
      "External id": 2484814,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768795.821, "dur": 6.998,
    "args": {
      "External id": 2484815,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768809.295, "dur": 9.905,
    "args": {
      "External id": 2484816,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768813.776, "dur": 5.002,
    "args": {
      "External id": 2484817,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768816.866, "dur": 0.655,
    "args": {
      "External id": 2484818,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768825.230, "dur": 31.233,
    "args": {
      "External id": 2484819,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768828.645, "dur": 0.715,
    "args": {
      "External id": 2484820,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768830.222, "dur": 1.936,
    "args": {
      "External id": 2484821,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768832.839, "dur": 0.729,
    "args": {
      "External id": 2484822,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768835.707, "dur": 0.272,
    "args": {
      "External id": 2484823,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768838.332, "dur": 0.328,
    "args": {
      "External id": 2484824,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768839.281, "dur": 2.460,
    "args": {
      "External id": 2484825,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768844.024, "dur": 0.569,
    "args": {
      "External id": 2484826,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768846.648, "dur": 0.662,
    "args": {
      "External id": 2484827,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768849.592, "dur": 0.515,
    "args": {
      "External id": 2484828,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768867.227, "dur": 23.868,
    "args": {
      "External id": 2484829,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656768966.841, "dur": 345.993,
    "args": {
      "External id": 2484830,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769042.146, "dur": 265.267,
    "args": {
      "External id": 2484831,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1630, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769056.589, "dur": 244.347,
    "args": {
      "External id": 2484832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769336.546, "dur": 2.639,
    "args": {
      "External id": 2484833,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1632, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769448.298, "dur": 1748.339,
    "args": {
      "External id": 2484834,"Sequence number": 24740476, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1633
    }
  },
  {
    "ph": "f", "id": 172, "pid": 1336760, "tid": 1381209, "ts": 1514656769448.298,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769602.014, "dur": 112.180,
    "args": {
      "External id": 2484835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769762.649, "dur": 43.403,
    "args": {
      "External id": 2484836,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769825.165, "dur": 49.293,
    "args": {
      "External id": 2484837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769883.936, "dur": 31.840,
    "args": {
      "External id": 2484838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769922.004, "dur": 45.571,
    "args": {
      "External id": 2484839,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656769976.035, "dur": 68.460,
    "args": {
      "External id": 2484840,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770056.679, "dur": 46.231,
    "args": {
      "External id": 2484841,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770131.732, "dur": 26.484,
    "args": {
      "External id": 2484842,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770179.087, "dur": 32.276,
    "args": {
      "External id": 2484843,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770241.940, "dur": 20.274,
    "args": {
      "External id": 2484844,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770277.771, "dur": 18.522,
    "args": {
      "External id": 2484845,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770305.544, "dur": 29.467,
    "args": {
      "External id": 2484846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770338.305, "dur": 32.377,
    "args": {
      "External id": 2484847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770400.454, "dur": 188.164,
    "args": {
      "External id": 2484848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770481.570, "dur": 6.768,
    "args": {
      "External id": 2484849,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770490.389, "dur": 2.185,
    "args": {
      "External id": 2484850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770631.819, "dur": 27.381,
    "args": {
      "External id": 2484851,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770676.132, "dur": 14.330,
    "args": {
      "External id": 2484852,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770698.633, "dur": 39.062,
    "args": {
      "External id": 2484853,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770742.937, "dur": 35.506,
    "args": {
      "External id": 2484854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770785.169, "dur": 19.827,
    "args": {
      "External id": 2484855,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770811.204, "dur": 33.695,
    "args": {
      "External id": 2484856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770849.832, "dur": 19.489,
    "args": {
      "External id": 2484857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770875.911, "dur": 29.114,
    "args": {
      "External id": 2484858,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770926.696, "dur": 23.711,
    "args": {
      "External id": 2484859,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656770982.166, "dur": 69.213,
    "args": {
      "External id": 2484860,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771079.080, "dur": 20.227,
    "args": {
      "External id": 2484861,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771115.749, "dur": 14.465,
    "args": {
      "External id": 2484862,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771151.494, "dur": 16.479,
    "args": {
      "External id": 2484863,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771246.346, "dur": 15.913,
    "args": {
      "External id": 2484864,"Record function id": 0, "Ev Idx": 1663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771249.971, "dur": 11.258,
    "args": {
      "External id": 2484865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771254.383, "dur": 5.812,
    "args": {
      "External id": 2484866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771255.932, "dur": 4.134,
    "args": {
      "External id": 2484867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771266.385, "dur": 7.119,
    "args": {
      "External id": 2484868,"Record function id": 0, "Ev Idx": 1667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771269.897, "dur": 3.103,
    "args": {
      "External id": 2484869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771271.040, "dur": 1.393,
    "args": {
      "External id": 2484870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771271.707, "dur": 0.664,
    "args": {
      "External id": 2484871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771276.926, "dur": 4.835,
    "args": {
      "External id": 2484872,"Record function id": 0, "Ev Idx": 1671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771278.502, "dur": 2.813,
    "args": {
      "External id": 2484873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771279.153, "dur": 1.616,
    "args": {
      "External id": 2484874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771279.888, "dur": 0.799,
    "args": {
      "External id": 2484875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771285.115, "dur": 5.725,
    "args": {
      "External id": 2484876,"Record function id": 0, "Ev Idx": 1675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771286.625, "dur": 3.786,
    "args": {
      "External id": 2484877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771287.248, "dur": 2.776,
    "args": {
      "External id": 2484878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771287.727, "dur": 2.204,
    "args": {
      "External id": 2484879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771294.117, "dur": 4.960,
    "args": {
      "External id": 2484880,"Record function id": 0, "Ev Idx": 1679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771295.460, "dur": 3.197,
    "args": {
      "External id": 2484881,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771296.576, "dur": 1.545,
    "args": {
      "External id": 2484882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771297.057, "dur": 0.980,
    "args": {
      "External id": 2484883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771302.323, "dur": 5.053,
    "args": {
      "External id": 2484884,"Record function id": 0, "Ev Idx": 1683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771303.783, "dur": 3.218,
    "args": {
      "External id": 2484885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771304.928, "dur": 1.501,
    "args": {
      "External id": 2484886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771305.702, "dur": 0.660,
    "args": {
      "External id": 2484887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771310.589, "dur": 4.030,
    "args": {
      "External id": 2484888,"Record function id": 0, "Ev Idx": 1687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771311.929, "dur": 2.263,
    "args": {
      "External id": 2484889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771312.447, "dur": 1.179,
    "args": {
      "External id": 2484890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771312.747, "dur": 0.812,
    "args": {
      "External id": 2484891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771317.773, "dur": 5.803,
    "args": {
      "External id": 2484892,"Record function id": 0, "Ev Idx": 1691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771319.098, "dur": 4.097,
    "args": {
      "External id": 2484893,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771319.817, "dur": 2.972,
    "args": {
      "External id": 2484894,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771321.956, "dur": 0.767,
    "args": {
      "External id": 2484895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771328.402, "dur": 4.164,
    "args": {
      "External id": 2484896,"Record function id": 0, "Ev Idx": 1695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771329.778, "dur": 2.350,
    "args": {
      "External id": 2484897,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771330.209, "dur": 1.501,
    "args": {
      "External id": 2484898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771330.758, "dur": 0.890,
    "args": {
      "External id": 2484899,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771336.982, "dur": 173450.475,
    "args": {
      "External id": 2484900,"Record function id": 0, "Sequence number": 24740475, "Fwd thread id": 1, "Ev Idx": 1699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771338.674, "dur": 173439.667,
    "args": {
      "External id": 2484901,"Sequence number": 24740475, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1700
    }
  },
  {
    "ph": "f", "id": 173, "pid": 1336760, "tid": 1381209, "ts": 1514656771338.674,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.27)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771370.881, "dur": 44.747,
    "args": {
      "External id": 2484902,"Record function id": 0, "Ev Idx": 1701
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.27)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771423.101, "dur": 91.045,
    "args": {
      "External id": 2484903,"Record function id": 0, "Ev Idx": 1702
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.27)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771521.615, "dur": 173248.873,
    "args": {
      "External id": 2484904,"Record function id": 0, "Ev Idx": 1703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771596.651, "dur": 7.659,
    "args": {
      "External id": 2484905,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771616.297, "dur": 5.392,
    "args": {
      "External id": 2484906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771640.422, "dur": 172233.070,
    "args": {
      "External id": 2484907,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771656.236, "dur": 172206.637,
    "args": {
      "External id": 2484908,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771737.944, "dur": 6.336,
    "args": {
      "External id": 2484909,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771753.386, "dur": 172066.362,
    "args": {
      "External id": 2484910,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771756.187, "dur": 172062.561,
    "args": {
      "External id": 2484911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771759.639, "dur": 9.351,
    "args": {
      "External id": 2484912,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656771770.634, "dur": 172044.846,
    "args": {
      "External id": 2484913,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656943974.719, "dur": 35.399,
    "args": {
      "External id": 2484914,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656943977.540, "dur": 31.704,
    "args": {
      "External id": 2484915,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944051.810, "dur": 375.504,
    "args": {
      "External id": 2484916,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944087.284, "dur": 334.833,
    "args": {
      "External id": 2484917,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1716, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944102.683, "dur": 312.639,
    "args": {
      "External id": 2484918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944452.083, "dur": 2.116,
    "args": {
      "External id": 2484919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1718, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944517.025, "dur": 7.894,
    "args": {
      "External id": 2484920,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944601.845, "dur": 2.001,
    "args": {
      "External id": 2484921,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944620.592, "dur": 1.631,
    "args": {
      "External id": 2484922,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944638.174, "dur": 1.155,
    "args": {
      "External id": 2484923,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944650.376, "dur": 1.097,
    "args": {
      "External id": 2484924,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944661.642, "dur": 1.047,
    "args": {
      "External id": 2484925,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944674.791, "dur": 1.564,
    "args": {
      "External id": 2484926,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944688.886, "dur": 2.826,
    "args": {
      "External id": 2484927,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944702.439, "dur": 0.997,
    "args": {
      "External id": 2484928,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944804.120, "dur": 2881.940,
    "args": {
      "External id": 2484929,"Record function id": 0, "Ev Idx": 1728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944824.023, "dur": 1127.979,
    "args": {
      "External id": 2484930,"Record function id": 0, "Ev Idx": 1729
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944839.890, "dur": 391.160,
    "args": {
      "External id": 2484931,"Record function id": 0, "Ev Idx": 1730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944933.210, "dur": 4.445,
    "args": {
      "External id": 2484932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944940.707, "dur": 1.473,
    "args": {
      "External id": 2484933,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944943.774, "dur": 1.400,
    "args": {
      "External id": 2484934,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944946.730, "dur": 2.865,
    "args": {
      "External id": 2484935,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944952.759, "dur": 1.029,
    "args": {
      "External id": 2484936,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944955.296, "dur": 1.257,
    "args": {
      "External id": 2484937,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944957.941, "dur": 2.871,
    "args": {
      "External id": 2484938,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944962.385, "dur": 1.094,
    "args": {
      "External id": 2484939,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944967.473, "dur": 1.056,
    "args": {
      "External id": 2484940,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656944969.860, "dur": 1.035,
    "args": {
      "External id": 2484941,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945025.202, "dur": 174.125,
    "args": {
      "External id": 2484942,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945046.394, "dur": 148.560,
    "args": {
      "External id": 2484943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945073.408, "dur": 13.540,
    "args": {
      "External id": 2484944,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945091.366, "dur": 73.878,
    "args": {
      "External id": 2484945,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945095.879, "dur": 68.995,
    "args": {
      "External id": 2484946,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945099.552, "dur": 9.860,
    "args": {
      "External id": 2484947,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945110.881, "dur": 53.259,
    "args": {
      "External id": 2484948,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1747
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.25", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945327.467, "dur": 616.230,
    "args": {
      "External id": 2484949,"Record function id": 0, "Ev Idx": 1748
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945345.044, "dur": 586.712,
    "args": {
      "External id": 2484950,"Record function id": 0, "Ev Idx": 1749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945409.379, "dur": 4.325,
    "args": {
      "External id": 2484951,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945429.604, "dur": 39.437,
    "args": {
      "External id": 2484952,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945435.862, "dur": 3.519,
    "args": {
      "External id": 2484953,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945440.987, "dur": 0.606,
    "args": {
      "External id": 2484954,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945442.483, "dur": 0.552,
    "args": {
      "External id": 2484955,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945447.221, "dur": 0.620,
    "args": {
      "External id": 2484956,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945448.550, "dur": 0.658,
    "args": {
      "External id": 2484957,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945449.761, "dur": 0.842,
    "args": {
      "External id": 2484958,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945454.675, "dur": 2.112,
    "args": {
      "External id": 2484959,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945457.424, "dur": 0.447,
    "args": {
      "External id": 2484960,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945460.273, "dur": 2.074,
    "args": {
      "External id": 2484961,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945490.832, "dur": 32.582,
    "args": {
      "External id": 2484962,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945573.462, "dur": 114.314,
    "args": {
      "External id": 2484963,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945583.786, "dur": 6.486,
    "args": {
      "External id": 2484964,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945595.720, "dur": 12.247,
    "args": {
      "External id": 2484965,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945600.180, "dur": 7.360,
    "args": {
      "External id": 2484966,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945605.129, "dur": 0.716,
    "args": {
      "External id": 2484967,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945614.573, "dur": 29.274,
    "args": {
      "External id": 2484968,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945616.212, "dur": 0.504,
    "args": {
      "External id": 2484969,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945619.572, "dur": 1.797,
    "args": {
      "External id": 2484970,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945621.998, "dur": 0.967,
    "args": {
      "External id": 2484971,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945623.532, "dur": 2.498,
    "args": {
      "External id": 2484972,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945630.078, "dur": 0.495,
    "args": {
      "External id": 2484973,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945631.135, "dur": 0.570,
    "args": {
      "External id": 2484974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945633.910, "dur": 0.266,
    "args": {
      "External id": 2484975,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945636.449, "dur": 0.399,
    "args": {
      "External id": 2484976,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945637.486, "dur": 0.404,
    "args": {
      "External id": 2484977,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945657.291, "dur": 22.142,
    "args": {
      "External id": 2484978,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945732.277, "dur": 133.866,
    "args": {
      "External id": 2484979,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945764.782, "dur": 97.948,
    "args": {
      "External id": 2484980,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1779, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945776.489, "dur": 82.237,
    "args": {
      "External id": 2484981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945883.471, "dur": 2.035,
    "args": {
      "External id": 2484982,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1781, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656945958.615, "dur": 1701.718,
    "args": {
      "External id": 2484983,"Sequence number": 24740474, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1782
    }
  },
  {
    "ph": "f", "id": 174, "pid": 1336760, "tid": 1381209, "ts": 1514656945958.615,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946115.172, "dur": 106.715,
    "args": {
      "External id": 2484984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946267.625, "dur": 44.455,
    "args": {
      "External id": 2484985,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946329.504, "dur": 47.691,
    "args": {
      "External id": 2484986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946386.171, "dur": 32.286,
    "args": {
      "External id": 2484987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946426.000, "dur": 47.982,
    "args": {
      "External id": 2484988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946480.530, "dur": 29.863,
    "args": {
      "External id": 2484989,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946518.019, "dur": 60.644,
    "args": {
      "External id": 2484990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946607.327, "dur": 23.889,
    "args": {
      "External id": 2484991,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946655.987, "dur": 28.275,
    "args": {
      "External id": 2484992,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946705.946, "dur": 18.959,
    "args": {
      "External id": 2484993,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946741.089, "dur": 14.477,
    "args": {
      "External id": 2484994,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946763.303, "dur": 32.034,
    "args": {
      "External id": 2484995,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946798.700, "dur": 32.537,
    "args": {
      "External id": 2484996,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946862.253, "dur": 205.249,
    "args": {
      "External id": 2484997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946942.234, "dur": 6.026,
    "args": {
      "External id": 2484998,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656946950.173, "dur": 2.083,
    "args": {
      "External id": 2484999,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947111.954, "dur": 25.605,
    "args": {
      "External id": 2485000,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947151.875, "dur": 13.795,
    "args": {
      "External id": 2485001,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947174.488, "dur": 54.875,
    "args": {
      "External id": 2485002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947239.950, "dur": 40.419,
    "args": {
      "External id": 2485003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947294.651, "dur": 20.550,
    "args": {
      "External id": 2485004,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947319.467, "dur": 30.038,
    "args": {
      "External id": 2485005,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947354.094, "dur": 19.055,
    "args": {
      "External id": 2485006,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947382.588, "dur": 28.523,
    "args": {
      "External id": 2485007,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947432.237, "dur": 26.123,
    "args": {
      "External id": 2485008,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947477.412, "dur": 29.495,
    "args": {
      "External id": 2485009,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947525.231, "dur": 37.390,
    "args": {
      "External id": 2485010,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947585.468, "dur": 14.564,
    "args": {
      "External id": 2485011,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947614.935, "dur": 15.420,
    "args": {
      "External id": 2485012,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947708.983, "dur": 15.172,
    "args": {
      "External id": 2485013,"Record function id": 0, "Ev Idx": 1812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947712.457, "dur": 10.693,
    "args": {
      "External id": 2485014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947716.710, "dur": 5.376,
    "args": {
      "External id": 2485015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947718.070, "dur": 3.933,
    "args": {
      "External id": 2485016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947727.987, "dur": 4.870,
    "args": {
      "External id": 2485017,"Record function id": 0, "Ev Idx": 1816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947729.224, "dur": 3.214,
    "args": {
      "External id": 2485018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947730.083, "dur": 1.905,
    "args": {
      "External id": 2485019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947730.824, "dur": 1.100,
    "args": {
      "External id": 2485020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947736.154, "dur": 6.334,
    "args": {
      "External id": 2485021,"Record function id": 0, "Ev Idx": 1820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947737.295, "dur": 4.760,
    "args": {
      "External id": 2485022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947738.162, "dur": 3.460,
    "args": {
      "External id": 2485023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947738.748, "dur": 2.780,
    "args": {
      "External id": 2485024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947745.814, "dur": 4.705,
    "args": {
      "External id": 2485025,"Record function id": 0, "Ev Idx": 1824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947747.159, "dur": 2.902,
    "args": {
      "External id": 2485026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947747.964, "dur": 1.418,
    "args": {
      "External id": 2485027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947748.310, "dur": 0.965,
    "args": {
      "External id": 2485028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947753.865, "dur": 4.797,
    "args": {
      "External id": 2485029,"Record function id": 0, "Ev Idx": 1828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947755.435, "dur": 2.821,
    "args": {
      "External id": 2485030,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947756.316, "dur": 1.293,
    "args": {
      "External id": 2485031,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947756.788, "dur": 0.755,
    "args": {
      "External id": 2485032,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947762.076, "dur": 6.835,
    "args": {
      "External id": 2485033,"Record function id": 0, "Ev Idx": 1832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947765.164, "dur": 3.255,
    "args": {
      "External id": 2485034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947766.135, "dur": 1.650,
    "args": {
      "External id": 2485035,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947766.876, "dur": 0.842,
    "args": {
      "External id": 2485036,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947772.258, "dur": 3.828,
    "args": {
      "External id": 2485037,"Record function id": 0, "Ev Idx": 1836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947773.357, "dur": 2.301,
    "args": {
      "External id": 2485038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947773.791, "dur": 1.074,
    "args": {
      "External id": 2485039,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947774.251, "dur": 0.549,
    "args": {
      "External id": 2485040,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947779.162, "dur": 5.840,
    "args": {
      "External id": 2485041,"Record function id": 0, "Ev Idx": 1840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947780.293, "dur": 4.295,
    "args": {
      "External id": 2485042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947780.910, "dur": 3.037,
    "args": {
      "External id": 2485043,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947783.119, "dur": 0.762,
    "args": {
      "External id": 2485044,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947788.122, "dur": 3.780,
    "args": {
      "External id": 2485045,"Record function id": 0, "Ev Idx": 1844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947789.080, "dur": 2.432,
    "args": {
      "External id": 2485046,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947789.605, "dur": 1.221,
    "args": {
      "External id": 2485047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947790.167, "dur": 0.599,
    "args": {
      "External id": 2485048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947795.766, "dur": 41246.597,
    "args": {
      "External id": 2485049,"Record function id": 0, "Sequence number": 24740473, "Fwd thread id": 1, "Ev Idx": 1848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947797.367, "dur": 41234.922,
    "args": {
      "External id": 2485050,"Sequence number": 24740473, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1849
    }
  },
  {
    "ph": "f", "id": 175, "pid": 1336760, "tid": 1381209, "ts": 1514656947797.367,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947825.046, "dur": 36.703,
    "args": {
      "External id": 2485051,"Record function id": 0, "Ev Idx": 1850
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947869.309, "dur": 64.936,
    "args": {
      "External id": 2485052,"Record function id": 0, "Ev Idx": 1851
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.26)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656947942.043, "dur": 41080.786,
    "args": {
      "External id": 2485053,"Record function id": 0, "Ev Idx": 1852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948071.917, "dur": 10.069,
    "args": {
      "External id": 2485054,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948092.722, "dur": 6.821,
    "args": {
      "External id": 2485055,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948118.034, "dur": 39999.084,
    "args": {
      "External id": 2485056,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948131.636, "dur": 39975.055,
    "args": {
      "External id": 2485057,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948195.052, "dur": 14.555,
    "args": {
      "External id": 2485058,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948217.045, "dur": 39852.480,
    "args": {
      "External id": 2485059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948219.517, "dur": 39849.194,
    "args": {
      "External id": 2485060,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948225.335, "dur": 4.853,
    "args": {
      "External id": 2485061,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656948231.898, "dur": 39832.673,
    "args": {
      "External id": 2485062,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988218.717, "dur": 9.762,
    "args": {
      "External id": 2485063,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988221.776, "dur": 6.351,
    "args": {
      "External id": 2485064,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988257.071, "dur": 429.443,
    "args": {
      "External id": 2485065,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988290.910, "dur": 389.667,
    "args": {
      "External id": 2485066,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1865, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988305.137, "dur": 369.115,
    "args": {
      "External id": 2485067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988710.771, "dur": 2.510,
    "args": {
      "External id": 2485068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1867, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988779.307, "dur": 6.505,
    "args": {
      "External id": 2485069,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988831.937, "dur": 1.340,
    "args": {
      "External id": 2485070,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988849.091, "dur": 3.526,
    "args": {
      "External id": 2485071,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988864.704, "dur": 1.013,
    "args": {
      "External id": 2485072,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988876.777, "dur": 0.906,
    "args": {
      "External id": 2485073,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988888.910, "dur": 0.706,
    "args": {
      "External id": 2485074,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988899.475, "dur": 3.011,
    "args": {
      "External id": 2485075,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988912.716, "dur": 2.614,
    "args": {
      "External id": 2485076,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656988927.660, "dur": 0.856,
    "args": {
      "External id": 2485077,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989059.571, "dur": 2799.550,
    "args": {
      "External id": 2485078,"Record function id": 0, "Ev Idx": 1877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989080.387, "dur": 1083.694,
    "args": {
      "External id": 2485079,"Record function id": 0, "Ev Idx": 1878
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989096.007, "dur": 330.540,
    "args": {
      "External id": 2485080,"Record function id": 0, "Ev Idx": 1879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989187.564, "dur": 4.566,
    "args": {
      "External id": 2485081,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989195.084, "dur": 1.124,
    "args": {
      "External id": 2485082,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989197.826, "dur": 2.784,
    "args": {
      "External id": 2485083,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989201.938, "dur": 1.164,
    "args": {
      "External id": 2485084,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989206.155, "dur": 1.260,
    "args": {
      "External id": 2485085,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989208.693, "dur": 1.214,
    "args": {
      "External id": 2485086,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989213.055, "dur": 1.908,
    "args": {
      "External id": 2485087,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989216.254, "dur": 1.287,
    "args": {
      "External id": 2485088,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989220.272, "dur": 0.934,
    "args": {
      "External id": 2485089,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989222.418, "dur": 0.998,
    "args": {
      "External id": 2485090,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989240.068, "dur": 157.135,
    "args": {
      "External id": 2485091,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989258.811, "dur": 134.082,
    "args": {
      "External id": 2485092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989278.562, "dur": 15.401,
    "args": {
      "External id": 2485093,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989297.418, "dur": 68.149,
    "args": {
      "External id": 2485094,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989301.859, "dur": 63.348,
    "args": {
      "External id": 2485095,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989305.376, "dur": 4.960,
    "args": {
      "External id": 2485096,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989312.085, "dur": 52.326,
    "args": {
      "External id": 2485097,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1896
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.24", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989511.788, "dur": 643.742,
    "args": {
      "External id": 2485098,"Record function id": 0, "Ev Idx": 1897
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989544.559, "dur": 598.995,
    "args": {
      "External id": 2485099,"Record function id": 0, "Ev Idx": 1898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989612.088, "dur": 5.423,
    "args": {
      "External id": 2485100,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989634.224, "dur": 36.339,
    "args": {
      "External id": 2485101,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989639.942, "dur": 1.454,
    "args": {
      "External id": 2485102,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989644.690, "dur": 0.343,
    "args": {
      "External id": 2485103,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989645.734, "dur": 0.386,
    "args": {
      "External id": 2485104,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989648.363, "dur": 1.711,
    "args": {
      "External id": 2485105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989650.754, "dur": 0.404,
    "args": {
      "External id": 2485106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989653.366, "dur": 2.421,
    "args": {
      "External id": 2485107,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989658.351, "dur": 0.423,
    "args": {
      "External id": 2485108,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989659.366, "dur": 0.660,
    "args": {
      "External id": 2485109,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989662.491, "dur": 0.584,
    "args": {
      "External id": 2485110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989681.656, "dur": 32.776,
    "args": {
      "External id": 2485111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989744.641, "dur": 115.775,
    "args": {
      "External id": 2485112,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989754.236, "dur": 3.730,
    "args": {
      "External id": 2485113,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989762.778, "dur": 12.748,
    "args": {
      "External id": 2485114,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989767.073, "dur": 8.018,
    "args": {
      "External id": 2485115,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989771.518, "dur": 2.316,
    "args": {
      "External id": 2485116,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989781.888, "dur": 38.697,
    "args": {
      "External id": 2485117,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989783.211, "dur": 0.661,
    "args": {
      "External id": 2485118,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989787.119, "dur": 0.346,
    "args": {
      "External id": 2485119,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989798.106, "dur": 2.153,
    "args": {
      "External id": 2485120,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989804.440, "dur": 0.262,
    "args": {
      "External id": 2485121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989805.204, "dur": 0.251,
    "args": {
      "External id": 2485122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989805.924, "dur": 2.053,
    "args": {
      "External id": 2485123,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989810.057, "dur": 0.294,
    "args": {
      "External id": 2485124,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989810.788, "dur": 0.272,
    "args": {
      "External id": 2485125,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989813.685, "dur": 0.312,
    "args": {
      "External id": 2485126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989831.984, "dur": 20.627,
    "args": {
      "External id": 2485127,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989903.697, "dur": 165.639,
    "args": {
      "External id": 2485128,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989933.616, "dur": 131.942,
    "args": {
      "External id": 2485129,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1928, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514656989943.011, "dur": 117.386,
    "args": {
      "External id": 2485130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990087.422, "dur": 2.045,
    "args": {
      "External id": 2485131,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1930, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990172.224, "dur": 1663.740,
    "args": {
      "External id": 2485132,"Sequence number": 24740472, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1931
    }
  },
  {
    "ph": "f", "id": 176, "pid": 1336760, "tid": 1381209, "ts": 1514656990172.224,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990286.387, "dur": 103.699,
    "args": {
      "External id": 2485133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990431.050, "dur": 40.220,
    "args": {
      "External id": 2485134,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990487.516, "dur": 66.660,
    "args": {
      "External id": 2485135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990566.975, "dur": 36.709,
    "args": {
      "External id": 2485136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990611.593, "dur": 45.276,
    "args": {
      "External id": 2485137,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990664.936, "dur": 27.118,
    "args": {
      "External id": 2485138,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990708.090, "dur": 44.852,
    "args": {
      "External id": 2485139,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990785.480, "dur": 23.780,
    "args": {
      "External id": 2485140,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990828.817, "dur": 30.237,
    "args": {
      "External id": 2485141,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990881.113, "dur": 24.611,
    "args": {
      "External id": 2485142,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990922.856, "dur": 14.875,
    "args": {
      "External id": 2485143,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990947.320, "dur": 28.818,
    "args": {
      "External id": 2485144,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656990979.216, "dur": 73.865,
    "args": {
      "External id": 2485145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991086.467, "dur": 168.971,
    "args": {
      "External id": 2485146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991164.399, "dur": 9.526,
    "args": {
      "External id": 2485147,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991175.769, "dur": 2.292,
    "args": {
      "External id": 2485148,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991294.247, "dur": 26.585,
    "args": {
      "External id": 2485149,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991334.968, "dur": 13.937,
    "args": {
      "External id": 2485150,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991356.915, "dur": 39.451,
    "args": {
      "External id": 2485151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991401.450, "dur": 35.549,
    "args": {
      "External id": 2485152,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991443.947, "dur": 20.660,
    "args": {
      "External id": 2485153,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991470.964, "dur": 43.183,
    "args": {
      "External id": 2485154,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991543.780, "dur": 30.252,
    "args": {
      "External id": 2485155,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991583.650, "dur": 34.108,
    "args": {
      "External id": 2485156,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991641.062, "dur": 29.571,
    "args": {
      "External id": 2485157,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991694.291, "dur": 24.000,
    "args": {
      "External id": 2485158,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991733.053, "dur": 18.070,
    "args": {
      "External id": 2485159,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991766.849, "dur": 12.612,
    "args": {
      "External id": 2485160,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991793.213, "dur": 14.541,
    "args": {
      "External id": 2485161,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991881.768, "dur": 14.607,
    "args": {
      "External id": 2485162,"Record function id": 0, "Ev Idx": 1961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991884.838, "dur": 10.671,
    "args": {
      "External id": 2485163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991889.051, "dur": 5.576,
    "args": {
      "External id": 2485164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991890.476, "dur": 4.020,
    "args": {
      "External id": 2485165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991900.619, "dur": 6.663,
    "args": {
      "External id": 2485166,"Record function id": 0, "Ev Idx": 1965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991903.721, "dur": 3.115,
    "args": {
      "External id": 2485167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991904.869, "dur": 1.502,
    "args": {
      "External id": 2485168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991905.350, "dur": 0.926,
    "args": {
      "External id": 2485169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991910.656, "dur": 4.606,
    "args": {
      "External id": 2485170,"Record function id": 0, "Ev Idx": 1969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991911.896, "dur": 2.975,
    "args": {
      "External id": 2485171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991912.600, "dur": 1.731,
    "args": {
      "External id": 2485172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991913.444, "dur": 0.800,
    "args": {
      "External id": 2485173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991918.522, "dur": 3.854,
    "args": {
      "External id": 2485174,"Record function id": 0, "Ev Idx": 1973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991919.797, "dur": 2.172,
    "args": {
      "External id": 2485175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991920.367, "dur": 1.194,
    "args": {
      "External id": 2485176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991920.853, "dur": 0.646,
    "args": {
      "External id": 2485177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991925.468, "dur": 3.808,
    "args": {
      "External id": 2485178,"Record function id": 0, "Ev Idx": 1977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991926.690, "dur": 2.194,
    "args": {
      "External id": 2485179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991927.132, "dur": 1.119,
    "args": {
      "External id": 2485180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991927.404, "dur": 0.779,
    "args": {
      "External id": 2485181,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991932.428, "dur": 5.857,
    "args": {
      "External id": 2485182,"Record function id": 0, "Ev Idx": 1981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991933.608, "dur": 4.279,
    "args": {
      "External id": 2485183,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991934.548, "dur": 2.809,
    "args": {
      "External id": 2485184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991934.921, "dur": 2.370,
    "args": {
      "External id": 2485185,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991941.573, "dur": 5.878,
    "args": {
      "External id": 2485186,"Record function id": 0, "Ev Idx": 1985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991942.664, "dur": 4.346,
    "args": {
      "External id": 2485187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991943.319, "dur": 3.273,
    "args": {
      "External id": 2485188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991945.617, "dur": 0.911,
    "args": {
      "External id": 2485189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991950.629, "dur": 3.706,
    "args": {
      "External id": 2485190,"Record function id": 0, "Ev Idx": 1989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991951.711, "dur": 2.238,
    "args": {
      "External id": 2485191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991952.133, "dur": 1.125,
    "args": {
      "External id": 2485192,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991952.544, "dur": 0.652,
    "args": {
      "External id": 2485193,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991957.398, "dur": 3.939,
    "args": {
      "External id": 2485194,"Record function id": 0, "Ev Idx": 1993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991958.645, "dur": 2.290,
    "args": {
      "External id": 2485195,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991959.496, "dur": 1.025,
    "args": {
      "External id": 2485196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991959.752, "dur": 0.707,
    "args": {
      "External id": 2485197,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991965.225, "dur": 37507.662,
    "args": {
      "External id": 2485198,"Record function id": 0, "Sequence number": 24740471, "Fwd thread id": 1, "Ev Idx": 1997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514656991966.750, "dur": 37498.178,
    "args": {
      "External id": 2485199,"Sequence number": 24740471, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1998
    }
  },
  {
    "ph": "f", "id": 177, "pid": 1336760, "tid": 1381209, "ts": 1514656991966.750,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992035.250, "dur": 39.044,
    "args": {
      "External id": 2485200,"Record function id": 0, "Ev Idx": 1999
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992082.154, "dur": 68.570,
    "args": {
      "External id": 2485201,"Record function id": 0, "Ev Idx": 2000
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.25)", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992160.358, "dur": 37297.340,
    "args": {
      "External id": 2485202,"Record function id": 0, "Ev Idx": 2001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992249.408, "dur": 6.982,
    "args": {
      "External id": 2485203,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992268.583, "dur": 5.119,
    "args": {
      "External id": 2485204,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992286.701, "dur": 36362.863,
    "args": {
      "External id": 2485205,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992299.882, "dur": 36340.975,
    "args": {
      "External id": 2485206,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992360.829, "dur": 13.935,
    "args": {
      "External id": 2485207,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992385.105, "dur": 36218.414,
    "args": {
      "External id": 2485208,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992387.399, "dur": 36215.464,
    "args": {
      "External id": 2485209,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992390.899, "dur": 6.335,
    "args": {
      "External id": 2485210,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514656992398.817, "dur": 36200.482,
    "args": {
      "External id": 2485211,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657028737.704, "dur": 8.844,
    "args": {
      "External id": 2485212,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657028740.742, "dur": 5.486,
    "args": {
      "External id": 2485213,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657028775.924, "dur": 390.902,
    "args": {
      "External id": 2485214,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657028808.215, "dur": 352.748,
    "args": {
      "External id": 2485215,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2014, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657028820.794, "dur": 331.715,
    "args": {
      "External id": 2485216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029192.990, "dur": 2.594,
    "args": {
      "External id": 2485217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2016, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029258.330, "dur": 8.088,
    "args": {
      "External id": 2485218,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029309.372, "dur": 1.882,
    "args": {
      "External id": 2485219,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029325.852, "dur": 1.171,
    "args": {
      "External id": 2485220,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029338.965, "dur": 1.093,
    "args": {
      "External id": 2485221,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029350.367, "dur": 2.798,
    "args": {
      "External id": 2485222,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029362.267, "dur": 1.181,
    "args": {
      "External id": 2485223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029375.967, "dur": 1.260,
    "args": {
      "External id": 2485224,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029387.547, "dur": 2.730,
    "args": {
      "External id": 2485225,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029399.302, "dur": 2.745,
    "args": {
      "External id": 2485226,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029487.048, "dur": 2876.357,
    "args": {
      "External id": 2485227,"Record function id": 0, "Ev Idx": 2026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029506.592, "dur": 1127.036,
    "args": {
      "External id": 2485228,"Record function id": 0, "Ev Idx": 2027
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029520.485, "dur": 349.875,
    "args": {
      "External id": 2485229,"Record function id": 0, "Ev Idx": 2028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029627.324, "dur": 4.640,
    "args": {
      "External id": 2485230,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029635.151, "dur": 1.069,
    "args": {
      "External id": 2485231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029639.824, "dur": 1.204,
    "args": {
      "External id": 2485232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029642.472, "dur": 1.267,
    "args": {
      "External id": 2485233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029645.510, "dur": 1.186,
    "args": {
      "External id": 2485234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029648.217, "dur": 0.916,
    "args": {
      "External id": 2485235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029652.266, "dur": 4.787,
    "args": {
      "External id": 2485236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029658.400, "dur": 1.147,
    "args": {
      "External id": 2485237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029661.171, "dur": 0.916,
    "args": {
      "External id": 2485238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029663.390, "dur": 0.677,
    "args": {
      "External id": 2485239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029683.336, "dur": 153.869,
    "args": {
      "External id": 2485240,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029700.921, "dur": 131.727,
    "args": {
      "External id": 2485241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029721.175, "dur": 13.790,
    "args": {
      "External id": 2485242,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029738.289, "dur": 67.450,
    "args": {
      "External id": 2485243,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029740.676, "dur": 64.701,
    "args": {
      "External id": 2485244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029744.682, "dur": 7.153,
    "args": {
      "External id": 2485245,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029753.400, "dur": 51.407,
    "args": {
      "External id": 2485246,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2045
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.23", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029958.643, "dur": 667.442,
    "args": {
      "External id": 2485247,"Record function id": 0, "Ev Idx": 2046
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657029975.323, "dur": 637.868,
    "args": {
      "External id": 2485248,"Record function id": 0, "Ev Idx": 2047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030102.307, "dur": 6.786,
    "args": {
      "External id": 2485249,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030126.402, "dur": 40.443,
    "args": {
      "External id": 2485250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030130.957, "dur": 3.454,
    "args": {
      "External id": 2485251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030136.283, "dur": 2.785,
    "args": {
      "External id": 2485252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030140.727, "dur": 0.273,
    "args": {
      "External id": 2485253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030144.182, "dur": 3.687,
    "args": {
      "External id": 2485254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030149.276, "dur": 0.306,
    "args": {
      "External id": 2485255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030150.817, "dur": 0.363,
    "args": {
      "External id": 2485256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030153.941, "dur": 0.267,
    "args": {
      "External id": 2485257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030155.777, "dur": 0.478,
    "args": {
      "External id": 2485258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030157.546, "dur": 2.022,
    "args": {
      "External id": 2485259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030177.322, "dur": 39.621,
    "args": {
      "External id": 2485260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030250.315, "dur": 107.183,
    "args": {
      "External id": 2485261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030260.463, "dur": 5.220,
    "args": {
      "External id": 2485262,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030270.757, "dur": 10.087,
    "args": {
      "External id": 2485263,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030274.941, "dur": 5.500,
    "args": {
      "External id": 2485264,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030278.465, "dur": 0.609,
    "args": {
      "External id": 2485265,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030287.667, "dur": 29.600,
    "args": {
      "External id": 2485266,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030289.768, "dur": 0.421,
    "args": {
      "External id": 2485267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030291.726, "dur": 0.247,
    "args": {
      "External id": 2485268,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030293.390, "dur": 2.389,
    "args": {
      "External id": 2485269,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030297.195, "dur": 0.482,
    "args": {
      "External id": 2485270,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030298.699, "dur": 0.253,
    "args": {
      "External id": 2485271,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030302.665, "dur": 0.238,
    "args": {
      "External id": 2485272,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030303.851, "dur": 2.053,
    "args": {
      "External id": 2485273,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030307.376, "dur": 0.340,
    "args": {
      "External id": 2485274,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030310.259, "dur": 0.354,
    "args": {
      "External id": 2485275,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030331.642, "dur": 18.212,
    "args": {
      "External id": 2485276,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030400.712, "dur": 120.983,
    "args": {
      "External id": 2485277,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030429.629, "dur": 88.788,
    "args": {
      "External id": 2485278,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2077, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030439.517, "dur": 74.730,
    "args": {
      "External id": 2485279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030556.129, "dur": 2.935,
    "args": {
      "External id": 2485280,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2079, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030640.861, "dur": 1701.578,
    "args": {
      "External id": 2485281,"Sequence number": 24740470, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2080
    }
  },
  {
    "ph": "f", "id": 178, "pid": 1336760, "tid": 1381209, "ts": 1514657030640.861,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030752.034, "dur": 104.300,
    "args": {
      "External id": 2485282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030897.117, "dur": 40.069,
    "args": {
      "External id": 2485283,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657030955.410, "dur": 90.639,
    "args": {
      "External id": 2485284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031059.422, "dur": 36.834,
    "args": {
      "External id": 2485285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031102.820, "dur": 46.293,
    "args": {
      "External id": 2485286,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031156.224, "dur": 27.059,
    "args": {
      "External id": 2485287,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031191.727, "dur": 40.879,
    "args": {
      "External id": 2485288,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031260.335, "dur": 23.804,
    "args": {
      "External id": 2485289,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031305.279, "dur": 28.272,
    "args": {
      "External id": 2485290,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031353.341, "dur": 24.385,
    "args": {
      "External id": 2485291,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031394.505, "dur": 14.573,
    "args": {
      "External id": 2485292,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031419.712, "dur": 28.204,
    "args": {
      "External id": 2485293,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031451.133, "dur": 31.431,
    "args": {
      "External id": 2485294,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031514.194, "dur": 192.716,
    "args": {
      "External id": 2485295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031614.021, "dur": 6.490,
    "args": {
      "External id": 2485296,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031622.661, "dur": 2.233,
    "args": {
      "External id": 2485297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031743.330, "dur": 23.884,
    "args": {
      "External id": 2485298,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031778.929, "dur": 13.803,
    "args": {
      "External id": 2485299,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031800.192, "dur": 38.763,
    "args": {
      "External id": 2485300,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031846.203, "dur": 34.176,
    "args": {
      "External id": 2485301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031887.964, "dur": 20.180,
    "args": {
      "External id": 2485302,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031912.044, "dur": 29.107,
    "args": {
      "External id": 2485303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031946.385, "dur": 19.676,
    "args": {
      "External id": 2485304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657031975.784, "dur": 80.496,
    "args": {
      "External id": 2485305,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032088.358, "dur": 27.860,
    "args": {
      "External id": 2485306,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032181.162, "dur": 26.427,
    "args": {
      "External id": 2485307,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032226.759, "dur": 16.750,
    "args": {
      "External id": 2485308,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032263.409, "dur": 17.836,
    "args": {
      "External id": 2485309,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032294.774, "dur": 17.149,
    "args": {
      "External id": 2485310,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032386.692, "dur": 20.295,
    "args": {
      "External id": 2485311,"Record function id": 0, "Ev Idx": 2110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032390.550, "dur": 15.488,
    "args": {
      "External id": 2485312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032398.163, "dur": 6.807,
    "args": {
      "External id": 2485313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032399.598, "dur": 5.238,
    "args": {
      "External id": 2485314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032410.995, "dur": 4.878,
    "args": {
      "External id": 2485315,"Record function id": 0, "Ev Idx": 2114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032412.529, "dur": 2.924,
    "args": {
      "External id": 2485316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032413.399, "dur": 1.502,
    "args": {
      "External id": 2485317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032413.965, "dur": 0.850,
    "args": {
      "External id": 2485318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032419.238, "dur": 4.158,
    "args": {
      "External id": 2485319,"Record function id": 0, "Ev Idx": 2118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032420.616, "dur": 2.374,
    "args": {
      "External id": 2485320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032421.062, "dur": 1.263,
    "args": {
      "External id": 2485321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032421.632, "dur": 0.616,
    "args": {
      "External id": 2485322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032426.633, "dur": 3.808,
    "args": {
      "External id": 2485323,"Record function id": 0, "Ev Idx": 2122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032427.766, "dur": 2.280,
    "args": {
      "External id": 2485324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032428.397, "dur": 0.964,
    "args": {
      "External id": 2485325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032428.686, "dur": 0.608,
    "args": {
      "External id": 2485326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032433.900, "dur": 3.579,
    "args": {
      "External id": 2485327,"Record function id": 0, "Ev Idx": 2126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032435.033, "dur": 2.065,
    "args": {
      "External id": 2485328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032435.632, "dur": 1.048,
    "args": {
      "External id": 2485329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032436.010, "dur": 0.603,
    "args": {
      "External id": 2485330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032440.678, "dur": 3.651,
    "args": {
      "External id": 2485331,"Record function id": 0, "Ev Idx": 2130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032441.925, "dur": 2.009,
    "args": {
      "External id": 2485332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032442.387, "dur": 1.064,
    "args": {
      "External id": 2485333,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032442.740, "dur": 0.645,
    "args": {
      "External id": 2485334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032447.716, "dur": 5.477,
    "args": {
      "External id": 2485335,"Record function id": 0, "Ev Idx": 2134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032449.141, "dur": 3.665,
    "args": {
      "External id": 2485336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032449.598, "dur": 2.667,
    "args": {
      "External id": 2485337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032451.682, "dur": 0.516,
    "args": {
      "External id": 2485338,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032456.546, "dur": 5.177,
    "args": {
      "External id": 2485339,"Record function id": 0, "Ev Idx": 2138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032458.003, "dur": 3.315,
    "args": {
      "External id": 2485340,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032458.472, "dur": 2.458,
    "args": {
      "External id": 2485341,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032458.768, "dur": 2.052,
    "args": {
      "External id": 2485342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032464.812, "dur": 3.412,
    "args": {
      "External id": 2485343,"Record function id": 0, "Ev Idx": 2142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032465.896, "dur": 1.942,
    "args": {
      "External id": 2485344,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032466.329, "dur": 1.081,
    "args": {
      "External id": 2485345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032466.618, "dur": 0.726,
    "args": {
      "External id": 2485346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032472.116, "dur": 37643.359,
    "args": {
      "External id": 2485347,"Record function id": 0, "Sequence number": 24740469, "Fwd thread id": 1, "Ev Idx": 2146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032473.242, "dur": 37633.549,
    "args": {
      "External id": 2485348,"Sequence number": 24740469, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2147
    }
  },
  {
    "ph": "f", "id": 179, "pid": 1336760, "tid": 1381209, "ts": 1514657032473.242,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032500.574, "dur": 54.097,
    "args": {
      "External id": 2485349,"Record function id": 0, "Ev Idx": 2148
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032563.798, "dur": 67.208,
    "args": {
      "External id": 2485350,"Record function id": 0, "Ev Idx": 2149
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.24)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032638.798, "dur": 37460.103,
    "args": {
      "External id": 2485351,"Record function id": 0, "Ev Idx": 2150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032725.696, "dur": 6.764,
    "args": {
      "External id": 2485352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032742.872, "dur": 5.282,
    "args": {
      "External id": 2485353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032761.121, "dur": 36439.486,
    "args": {
      "External id": 2485354,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032774.271, "dur": 36417.315,
    "args": {
      "External id": 2485355,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032827.912, "dur": 13.213,
    "args": {
      "External id": 2485356,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032847.551, "dur": 36306.743,
    "args": {
      "External id": 2485357,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032852.047, "dur": 36301.537,
    "args": {
      "External id": 2485358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032856.198, "dur": 4.649,
    "args": {
      "External id": 2485359,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657032862.306, "dur": 36287.712,
    "args": {
      "External id": 2485360,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069292.813, "dur": 8.487,
    "args": {
      "External id": 2485361,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069295.885, "dur": 5.112,
    "args": {
      "External id": 2485362,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069330.867, "dur": 417.699,
    "args": {
      "External id": 2485363,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069361.348, "dur": 381.889,
    "args": {
      "External id": 2485364,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2163, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069372.947, "dur": 364.012,
    "args": {
      "External id": 2485365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069774.849, "dur": 5.538,
    "args": {
      "External id": 2485366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2165, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069844.619, "dur": 6.564,
    "args": {
      "External id": 2485367,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069896.513, "dur": 3.557,
    "args": {
      "External id": 2485368,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069916.059, "dur": 1.410,
    "args": {
      "External id": 2485369,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069930.001, "dur": 0.896,
    "args": {
      "External id": 2485370,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069943.838, "dur": 1.160,
    "args": {
      "External id": 2485371,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069956.788, "dur": 2.565,
    "args": {
      "External id": 2485372,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657069970.560, "dur": 1.150,
    "args": {
      "External id": 2485373,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070016.747, "dur": 3.344,
    "args": {
      "External id": 2485374,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070037.080, "dur": 1.230,
    "args": {
      "External id": 2485375,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070132.119, "dur": 2719.106,
    "args": {
      "External id": 2485376,"Record function id": 0, "Ev Idx": 2175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070151.499, "dur": 1053.087,
    "args": {
      "External id": 2485377,"Record function id": 0, "Ev Idx": 2176
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070165.148, "dur": 324.476,
    "args": {
      "External id": 2485378,"Record function id": 0, "Ev Idx": 2177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070251.106, "dur": 6.094,
    "args": {
      "External id": 2485379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070260.262, "dur": 1.243,
    "args": {
      "External id": 2485380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070263.528, "dur": 1.137,
    "args": {
      "External id": 2485381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070266.661, "dur": 0.955,
    "args": {
      "External id": 2485382,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070270.716, "dur": 0.773,
    "args": {
      "External id": 2485383,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070273.353, "dur": 0.990,
    "args": {
      "External id": 2485384,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070275.814, "dur": 2.578,
    "args": {
      "External id": 2485385,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070279.949, "dur": 1.079,
    "args": {
      "External id": 2485386,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070284.740, "dur": 3.137,
    "args": {
      "External id": 2485387,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070289.591, "dur": 0.850,
    "args": {
      "External id": 2485388,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070308.723, "dur": 150.925,
    "args": {
      "External id": 2485389,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070324.769, "dur": 130.553,
    "args": {
      "External id": 2485390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070346.708, "dur": 13.008,
    "args": {
      "External id": 2485391,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070362.991, "dur": 66.234,
    "args": {
      "External id": 2485392,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070365.615, "dur": 63.238,
    "args": {
      "External id": 2485393,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070369.492, "dur": 4.763,
    "args": {
      "External id": 2485394,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070377.610, "dur": 50.479,
    "args": {
      "External id": 2485395,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2194
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.22", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070602.251, "dur": 594.592,
    "args": {
      "External id": 2485396,"Record function id": 0, "Ev Idx": 2195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070619.588, "dur": 566.128,
    "args": {
      "External id": 2485397,"Record function id": 0, "Ev Idx": 2196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070681.031, "dur": 5.219,
    "args": {
      "External id": 2485398,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070703.147, "dur": 34.890,
    "args": {
      "External id": 2485399,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070707.841, "dur": 1.849,
    "args": {
      "External id": 2485400,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070712.397, "dur": 0.595,
    "args": {
      "External id": 2485401,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070714.710, "dur": 0.442,
    "args": {
      "External id": 2485402,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070716.568, "dur": 2.730,
    "args": {
      "External id": 2485403,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070721.156, "dur": 0.527,
    "args": {
      "External id": 2485404,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070722.833, "dur": 0.393,
    "args": {
      "External id": 2485405,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070726.582, "dur": 0.481,
    "args": {
      "External id": 2485406,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070728.449, "dur": 0.315,
    "args": {
      "External id": 2485407,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070730.582, "dur": 0.398,
    "args": {
      "External id": 2485408,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070748.740, "dur": 32.077,
    "args": {
      "External id": 2485409,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070811.506, "dur": 102.244,
    "args": {
      "External id": 2485410,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070821.012, "dur": 3.260,
    "args": {
      "External id": 2485411,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070829.776, "dur": 11.523,
    "args": {
      "External id": 2485412,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070833.931, "dur": 6.925,
    "args": {
      "External id": 2485413,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070837.553, "dur": 1.937,
    "args": {
      "External id": 2485414,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070848.390, "dur": 29.322,
    "args": {
      "External id": 2485415,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070850.294, "dur": 2.000,
    "args": {
      "External id": 2485416,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070854.216, "dur": 0.660,
    "args": {
      "External id": 2485417,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070856.439, "dur": 0.477,
    "args": {
      "External id": 2485418,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070858.956, "dur": 0.495,
    "args": {
      "External id": 2485419,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070860.763, "dur": 0.440,
    "args": {
      "External id": 2485420,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070862.273, "dur": 1.849,
    "args": {
      "External id": 2485421,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070865.520, "dur": 0.336,
    "args": {
      "External id": 2485422,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070867.057, "dur": 0.291,
    "args": {
      "External id": 2485423,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070870.489, "dur": 2.073,
    "args": {
      "External id": 2485424,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070887.439, "dur": 18.355,
    "args": {
      "External id": 2485425,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657070954.776, "dur": 156.732,
    "args": {
      "External id": 2485426,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071015.871, "dur": 92.120,
    "args": {
      "External id": 2485427,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2226, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071027.075, "dur": 76.400,
    "args": {
      "External id": 2485428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071131.289, "dur": 1.834,
    "args": {
      "External id": 2485429,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2228, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071212.513, "dur": 1614.610,
    "args": {
      "External id": 2485430,"Sequence number": 24740468, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2229
    }
  },
  {
    "ph": "f", "id": 180, "pid": 1336760, "tid": 1381209, "ts": 1514657071212.513,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071326.589, "dur": 105.602,
    "args": {
      "External id": 2485431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071471.799, "dur": 40.352,
    "args": {
      "External id": 2485432,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071549.016, "dur": 54.102,
    "args": {
      "External id": 2485433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071614.046, "dur": 32.222,
    "args": {
      "External id": 2485434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071652.444, "dur": 44.457,
    "args": {
      "External id": 2485435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071705.000, "dur": 27.465,
    "args": {
      "External id": 2485436,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071742.048, "dur": 41.901,
    "args": {
      "External id": 2485437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071810.952, "dur": 22.854,
    "args": {
      "External id": 2485438,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071851.841, "dur": 26.271,
    "args": {
      "External id": 2485439,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071899.930, "dur": 17.461,
    "args": {
      "External id": 2485440,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071931.362, "dur": 13.843,
    "args": {
      "External id": 2485441,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657071955.782, "dur": 64.748,
    "args": {
      "External id": 2485442,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072026.595, "dur": 36.949,
    "args": {
      "External id": 2485443,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072092.086, "dur": 170.255,
    "args": {
      "External id": 2485444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072169.866, "dur": 7.199,
    "args": {
      "External id": 2485445,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072179.243, "dur": 2.201,
    "args": {
      "External id": 2485446,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072298.127, "dur": 23.053,
    "args": {
      "External id": 2485447,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072334.205, "dur": 13.465,
    "args": {
      "External id": 2485448,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072355.433, "dur": 34.829,
    "args": {
      "External id": 2485449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072397.374, "dur": 33.952,
    "args": {
      "External id": 2485450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072448.436, "dur": 20.688,
    "args": {
      "External id": 2485451,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072473.849, "dur": 29.809,
    "args": {
      "External id": 2485452,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072509.904, "dur": 36.154,
    "args": {
      "External id": 2485453,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072558.651, "dur": 49.164,
    "args": {
      "External id": 2485454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072635.736, "dur": 23.263,
    "args": {
      "External id": 2485455,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072677.992, "dur": 24.676,
    "args": {
      "External id": 2485456,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072723.005, "dur": 15.650,
    "args": {
      "External id": 2485457,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072756.397, "dur": 17.185,
    "args": {
      "External id": 2485458,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072785.529, "dur": 14.660,
    "args": {
      "External id": 2485459,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072873.584, "dur": 17.417,
    "args": {
      "External id": 2485460,"Record function id": 0, "Ev Idx": 2259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072877.257, "dur": 12.738,
    "args": {
      "External id": 2485461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072881.439, "dur": 7.633,
    "args": {
      "External id": 2485462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072882.955, "dur": 6.000,
    "args": {
      "External id": 2485463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072895.475, "dur": 5.276,
    "args": {
      "External id": 2485464,"Record function id": 0, "Ev Idx": 2263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072897.237, "dur": 3.081,
    "args": {
      "External id": 2485465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072898.416, "dur": 1.420,
    "args": {
      "External id": 2485466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072898.783, "dur": 0.972,
    "args": {
      "External id": 2485467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072904.142, "dur": 4.680,
    "args": {
      "External id": 2485468,"Record function id": 0, "Ev Idx": 2267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072905.510, "dur": 2.889,
    "args": {
      "External id": 2485469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072906.224, "dur": 1.705,
    "args": {
      "External id": 2485470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072906.968, "dur": 0.842,
    "args": {
      "External id": 2485471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072912.039, "dur": 3.991,
    "args": {
      "External id": 2485472,"Record function id": 0, "Ev Idx": 2271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072913.341, "dur": 2.265,
    "args": {
      "External id": 2485473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072913.822, "dur": 1.140,
    "args": {
      "External id": 2485474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072914.262, "dur": 0.600,
    "args": {
      "External id": 2485475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072919.113, "dur": 4.039,
    "args": {
      "External id": 2485476,"Record function id": 0, "Ev Idx": 2275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072920.483, "dur": 2.241,
    "args": {
      "External id": 2485477,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072920.920, "dur": 1.203,
    "args": {
      "External id": 2485478,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072921.176, "dur": 0.879,
    "args": {
      "External id": 2485479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072926.341, "dur": 3.588,
    "args": {
      "External id": 2485480,"Record function id": 0, "Ev Idx": 2279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072927.481, "dur": 2.057,
    "args": {
      "External id": 2485481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072927.942, "dur": 1.065,
    "args": {
      "External id": 2485482,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072928.317, "dur": 0.624,
    "args": {
      "External id": 2485483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072933.345, "dur": 3.970,
    "args": {
      "External id": 2485484,"Record function id": 0, "Ev Idx": 2283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072934.641, "dur": 2.269,
    "args": {
      "External id": 2485485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072935.332, "dur": 1.145,
    "args": {
      "External id": 2485486,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072935.600, "dur": 0.812,
    "args": {
      "External id": 2485487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072940.473, "dur": 7.752,
    "args": {
      "External id": 2485488,"Record function id": 0, "Ev Idx": 2287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072941.801, "dur": 6.030,
    "args": {
      "External id": 2485489,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072942.227, "dur": 5.002,
    "args": {
      "External id": 2485490,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072944.626, "dur": 2.516,
    "args": {
      "External id": 2485491,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072951.334, "dur": 3.882,
    "args": {
      "External id": 2485492,"Record function id": 0, "Ev Idx": 2291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072952.687, "dur": 2.136,
    "args": {
      "External id": 2485493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072953.113, "dur": 1.120,
    "args": {
      "External id": 2485494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072953.410, "dur": 0.757,
    "args": {
      "External id": 2485495,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072958.673, "dur": 36800.324,
    "args": {
      "External id": 2485496,"Record function id": 0, "Sequence number": 24740467, "Fwd thread id": 1, "Ev Idx": 2295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657072960.001, "dur": 36790.449,
    "args": {
      "External id": 2485497,"Sequence number": 24740467, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2296
    }
  },
  {
    "ph": "f", "id": 181, "pid": 1336760, "tid": 1381209, "ts": 1514657072960.001,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073026.026, "dur": 40.387,
    "args": {
      "External id": 2485498,"Record function id": 0, "Ev Idx": 2297
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073075.133, "dur": 63.876,
    "args": {
      "External id": 2485499,"Record function id": 0, "Ev Idx": 2298
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.23)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073145.097, "dur": 36596.848,
    "args": {
      "External id": 2485500,"Record function id": 0, "Ev Idx": 2299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073234.041, "dur": 7.074,
    "args": {
      "External id": 2485501,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073251.507, "dur": 5.508,
    "args": {
      "External id": 2485502,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073272.000, "dur": 35625.956,
    "args": {
      "External id": 2485503,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073284.861, "dur": 35604.002,
    "args": {
      "External id": 2485504,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073333.803, "dur": 13.238,
    "args": {
      "External id": 2485505,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073353.199, "dur": 35496.880,
    "args": {
      "External id": 2485506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073355.667, "dur": 35493.657,
    "args": {
      "External id": 2485507,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073360.136, "dur": 5.019,
    "args": {
      "External id": 2485508,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657073367.194, "dur": 35478.682,
    "args": {
      "External id": 2485509,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109014.535, "dur": 9.629,
    "args": {
      "External id": 2485510,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109017.651, "dur": 6.026,
    "args": {
      "External id": 2485511,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109054.586, "dur": 354.970,
    "args": {
      "External id": 2485512,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109086.784, "dur": 318.101,
    "args": {
      "External id": 2485513,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2312, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109099.281, "dur": 300.223,
    "args": {
      "External id": 2485514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109429.116, "dur": 2.035,
    "args": {
      "External id": 2485515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2314, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109489.496, "dur": 6.334,
    "args": {
      "External id": 2485516,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109565.049, "dur": 1.872,
    "args": {
      "External id": 2485517,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109586.083, "dur": 1.351,
    "args": {
      "External id": 2485518,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109604.954, "dur": 0.853,
    "args": {
      "External id": 2485519,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109618.122, "dur": 0.789,
    "args": {
      "External id": 2485520,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109633.177, "dur": 1.165,
    "args": {
      "External id": 2485521,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109646.885, "dur": 1.001,
    "args": {
      "External id": 2485522,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109661.659, "dur": 2.217,
    "args": {
      "External id": 2485523,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109676.152, "dur": 1.010,
    "args": {
      "External id": 2485524,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109775.453, "dur": 2807.528,
    "args": {
      "External id": 2485525,"Record function id": 0, "Ev Idx": 2324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109795.832, "dur": 1079.193,
    "args": {
      "External id": 2485526,"Record function id": 0, "Ev Idx": 2325
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109811.310, "dur": 372.875,
    "args": {
      "External id": 2485527,"Record function id": 0, "Ev Idx": 2326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109895.615, "dur": 4.107,
    "args": {
      "External id": 2485528,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109903.008, "dur": 1.390,
    "args": {
      "External id": 2485529,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109906.521, "dur": 1.006,
    "args": {
      "External id": 2485530,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109909.313, "dur": 1.117,
    "args": {
      "External id": 2485531,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109912.324, "dur": 1.136,
    "args": {
      "External id": 2485532,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109915.440, "dur": 0.997,
    "args": {
      "External id": 2485533,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109918.313, "dur": 1.763,
    "args": {
      "External id": 2485534,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109921.429, "dur": 2.860,
    "args": {
      "External id": 2485535,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109925.867, "dur": 1.025,
    "args": {
      "External id": 2485536,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109928.349, "dur": 0.841,
    "args": {
      "External id": 2485537,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109947.279, "dur": 200.948,
    "args": {
      "External id": 2485538,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657109962.254, "dur": 181.027,
    "args": {
      "External id": 2485539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110024.023, "dur": 14.559,
    "args": {
      "External id": 2485540,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110042.271, "dur": 71.917,
    "args": {
      "External id": 2485541,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110045.320, "dur": 68.489,
    "args": {
      "External id": 2485542,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110049.607, "dur": 6.639,
    "args": {
      "External id": 2485543,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110058.214, "dur": 55.073,
    "args": {
      "External id": 2485544,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2343
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.21", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110278.996, "dur": 586.827,
    "args": {
      "External id": 2485545,"Record function id": 0, "Ev Idx": 2344
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110298.822, "dur": 554.601,
    "args": {
      "External id": 2485546,"Record function id": 0, "Ev Idx": 2345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110363.304, "dur": 8.527,
    "args": {
      "External id": 2485547,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110387.859, "dur": 32.474,
    "args": {
      "External id": 2485548,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110393.015, "dur": 1.772,
    "args": {
      "External id": 2485549,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110397.442, "dur": 0.652,
    "args": {
      "External id": 2485550,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110399.625, "dur": 2.167,
    "args": {
      "External id": 2485551,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110403.773, "dur": 0.624,
    "args": {
      "External id": 2485552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110406.169, "dur": 0.334,
    "args": {
      "External id": 2485553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110408.286, "dur": 0.390,
    "args": {
      "External id": 2485554,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110410.544, "dur": 0.343,
    "args": {
      "External id": 2485555,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110412.713, "dur": 0.336,
    "args": {
      "External id": 2485556,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110414.747, "dur": 0.443,
    "args": {
      "External id": 2485557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110435.031, "dur": 32.780,
    "args": {
      "External id": 2485558,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110502.032, "dur": 126.736,
    "args": {
      "External id": 2485559,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110512.343, "dur": 3.032,
    "args": {
      "External id": 2485560,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110520.477, "dur": 30.826,
    "args": {
      "External id": 2485561,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110524.925, "dur": 25.963,
    "args": {
      "External id": 2485562,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110545.717, "dur": 3.067,
    "args": {
      "External id": 2485563,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110560.204, "dur": 26.690,
    "args": {
      "External id": 2485564,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110563.154, "dur": 0.459,
    "args": {
      "External id": 2485565,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110565.347, "dur": 0.588,
    "args": {
      "External id": 2485566,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110567.616, "dur": 0.419,
    "args": {
      "External id": 2485567,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110569.823, "dur": 0.324,
    "args": {
      "External id": 2485568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110571.773, "dur": 0.542,
    "args": {
      "External id": 2485569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110573.902, "dur": 0.495,
    "args": {
      "External id": 2485570,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110575.998, "dur": 0.440,
    "args": {
      "External id": 2485571,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110578.069, "dur": 2.323,
    "args": {
      "External id": 2485572,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110582.127, "dur": 0.294,
    "args": {
      "External id": 2485573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110599.559, "dur": 21.712,
    "args": {
      "External id": 2485574,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110674.958, "dur": 112.314,
    "args": {
      "External id": 2485575,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110697.656, "dur": 86.150,
    "args": {
      "External id": 2485576,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2375, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110707.682, "dur": 71.993,
    "args": {
      "External id": 2485577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110802.512, "dur": 1.950,
    "args": {
      "External id": 2485578,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2377, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657110882.633, "dur": 1676.958,
    "args": {
      "External id": 2485579,"Sequence number": 24740466, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2378
    }
  },
  {
    "ph": "f", "id": 182, "pid": 1336760, "tid": 1381209, "ts": 1514657110882.633,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111029.901, "dur": 105.180,
    "args": {
      "External id": 2485580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111177.781, "dur": 45.999,
    "args": {
      "External id": 2485581,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111243.578, "dur": 49.254,
    "args": {
      "External id": 2485582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111302.807, "dur": 33.190,
    "args": {
      "External id": 2485583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111343.263, "dur": 45.452,
    "args": {
      "External id": 2485584,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111396.705, "dur": 27.772,
    "args": {
      "External id": 2485585,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111434.546, "dur": 43.363,
    "args": {
      "External id": 2485586,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111499.716, "dur": 21.286,
    "args": {
      "External id": 2485587,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111567.339, "dur": 31.142,
    "args": {
      "External id": 2485588,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111620.849, "dur": 19.540,
    "args": {
      "External id": 2485589,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111654.025, "dur": 15.346,
    "args": {
      "External id": 2485590,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111681.774, "dur": 32.862,
    "args": {
      "External id": 2485591,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111718.236, "dur": 33.222,
    "args": {
      "External id": 2485592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111777.794, "dur": 170.787,
    "args": {
      "External id": 2485593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111854.678, "dur": 5.462,
    "args": {
      "External id": 2485594,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657111862.490, "dur": 2.098,
    "args": {
      "External id": 2485595,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112018.263, "dur": 28.186,
    "args": {
      "External id": 2485596,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112059.253, "dur": 14.527,
    "args": {
      "External id": 2485597,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112086.042, "dur": 41.375,
    "args": {
      "External id": 2485598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112134.656, "dur": 35.213,
    "args": {
      "External id": 2485599,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112176.972, "dur": 21.296,
    "args": {
      "External id": 2485600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112203.497, "dur": 44.549,
    "args": {
      "External id": 2485601,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112263.135, "dur": 23.923,
    "args": {
      "External id": 2485602,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112295.762, "dur": 31.590,
    "args": {
      "External id": 2485603,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112348.692, "dur": 22.380,
    "args": {
      "External id": 2485604,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112387.791, "dur": 28.413,
    "args": {
      "External id": 2485605,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112432.154, "dur": 16.395,
    "args": {
      "External id": 2485606,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112463.238, "dur": 14.653,
    "args": {
      "External id": 2485607,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112497.379, "dur": 15.642,
    "args": {
      "External id": 2485608,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112607.321, "dur": 15.713,
    "args": {
      "External id": 2485609,"Record function id": 0, "Ev Idx": 2408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112610.880, "dur": 11.180,
    "args": {
      "External id": 2485610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112614.984, "dur": 6.025,
    "args": {
      "External id": 2485611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112616.656, "dur": 4.226,
    "args": {
      "External id": 2485612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112627.178, "dur": 6.041,
    "args": {
      "External id": 2485613,"Record function id": 0, "Ev Idx": 2412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112628.961, "dur": 3.793,
    "args": {
      "External id": 2485614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112630.154, "dur": 2.125,
    "args": {
      "External id": 2485615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112631.047, "dur": 1.165,
    "args": {
      "External id": 2485616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112636.560, "dur": 4.907,
    "args": {
      "External id": 2485617,"Record function id": 0, "Ev Idx": 2416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112637.954, "dur": 3.123,
    "args": {
      "External id": 2485618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112638.755, "dur": 1.774,
    "args": {
      "External id": 2485619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112639.493, "dur": 0.969,
    "args": {
      "External id": 2485620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112644.746, "dur": 4.544,
    "args": {
      "External id": 2485621,"Record function id": 0, "Ev Idx": 2420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112646.271, "dur": 2.612,
    "args": {
      "External id": 2485622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112646.945, "dur": 1.309,
    "args": {
      "External id": 2485623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112647.369, "dur": 0.820,
    "args": {
      "External id": 2485624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112652.516, "dur": 4.273,
    "args": {
      "External id": 2485625,"Record function id": 0, "Ev Idx": 2424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112654.013, "dur": 2.368,
    "args": {
      "External id": 2485626,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112654.466, "dur": 1.304,
    "args": {
      "External id": 2485627,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112655.056, "dur": 0.648,
    "args": {
      "External id": 2485628,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112659.951, "dur": 4.308,
    "args": {
      "External id": 2485629,"Record function id": 0, "Ev Idx": 2428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112661.514, "dur": 2.318,
    "args": {
      "External id": 2485630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112662.165, "dur": 1.240,
    "args": {
      "External id": 2485631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112662.653, "dur": 0.650,
    "args": {
      "External id": 2485632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112667.531, "dur": 5.795,
    "args": {
      "External id": 2485633,"Record function id": 0, "Ev Idx": 2432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112669.148, "dur": 3.766,
    "args": {
      "External id": 2485634,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112669.629, "dur": 2.716,
    "args": {
      "External id": 2485635,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112669.946, "dur": 2.320,
    "args": {
      "External id": 2485636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112676.489, "dur": 3.859,
    "args": {
      "External id": 2485637,"Record function id": 0, "Ev Idx": 2436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112677.754, "dur": 2.171,
    "args": {
      "External id": 2485638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112678.386, "dur": 1.004,
    "args": {
      "External id": 2485639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112678.672, "dur": 0.637,
    "args": {
      "External id": 2485640,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112683.364, "dur": 4.022,
    "args": {
      "External id": 2485641,"Record function id": 0, "Ev Idx": 2440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112684.618, "dur": 2.387,
    "args": {
      "External id": 2485642,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112685.250, "dur": 1.181,
    "args": {
      "External id": 2485643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112685.668, "dur": 0.650,
    "args": {
      "External id": 2485644,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112690.928, "dur": 36162.752,
    "args": {
      "External id": 2485645,"Record function id": 0, "Sequence number": 24740465, "Fwd thread id": 1, "Ev Idx": 2444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112692.254, "dur": 36151.969,
    "args": {
      "External id": 2485646,"Sequence number": 24740465, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2445
    }
  },
  {
    "ph": "f", "id": 183, "pid": 1336760, "tid": 1381209, "ts": 1514657112692.254,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112721.935, "dur": 36.065,
    "args": {
      "External id": 2485647,"Record function id": 0, "Ev Idx": 2446
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112765.120, "dur": 64.728,
    "args": {
      "External id": 2485648,"Record function id": 0, "Ev Idx": 2447
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.22)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112835.554, "dur": 36001.342,
    "args": {
      "External id": 2485649,"Record function id": 0, "Ev Idx": 2448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112922.486, "dur": 6.206,
    "args": {
      "External id": 2485650,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112938.053, "dur": 4.843,
    "args": {
      "External id": 2485651,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112956.524, "dur": 35062.860,
    "args": {
      "External id": 2485652,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657112969.645, "dur": 35039.630,
    "args": {
      "External id": 2485653,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657113061.097, "dur": 14.588,
    "args": {
      "External id": 2485654,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657113082.042, "dur": 34861.838,
    "args": {
      "External id": 2485655,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657113084.778, "dur": 34858.177,
    "args": {
      "External id": 2485656,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657113089.000, "dur": 6.470,
    "args": {
      "External id": 2485657,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657113097.441, "dur": 34842.231,
    "args": {
      "External id": 2485658,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148111.179, "dur": 8.975,
    "args": {
      "External id": 2485659,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148114.385, "dur": 5.430,
    "args": {
      "External id": 2485660,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148152.063, "dur": 364.973,
    "args": {
      "External id": 2485661,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148179.807, "dur": 332.437,
    "args": {
      "External id": 2485662,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2461, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148192.363, "dur": 314.022,
    "args": {
      "External id": 2485663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148552.726, "dur": 3.399,
    "args": {
      "External id": 2485664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2463, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148617.710, "dur": 8.480,
    "args": {
      "External id": 2485665,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148672.268, "dur": 1.342,
    "args": {
      "External id": 2485666,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148691.356, "dur": 1.048,
    "args": {
      "External id": 2485667,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148705.699, "dur": 1.158,
    "args": {
      "External id": 2485668,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148720.245, "dur": 2.707,
    "args": {
      "External id": 2485669,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148735.172, "dur": 1.016,
    "args": {
      "External id": 2485670,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148747.984, "dur": 1.217,
    "args": {
      "External id": 2485671,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148761.517, "dur": 2.359,
    "args": {
      "External id": 2485672,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148775.394, "dur": 2.513,
    "args": {
      "External id": 2485673,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148870.043, "dur": 2822.194,
    "args": {
      "External id": 2485674,"Record function id": 0, "Ev Idx": 2473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148889.635, "dur": 1065.662,
    "args": {
      "External id": 2485675,"Record function id": 0, "Ev Idx": 2474
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657148903.750, "dur": 386.063,
    "args": {
      "External id": 2485676,"Record function id": 0, "Ev Idx": 2475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149032.273, "dur": 5.102,
    "args": {
      "External id": 2485677,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149041.168, "dur": 1.150,
    "args": {
      "External id": 2485678,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149044.222, "dur": 1.199,
    "args": {
      "External id": 2485679,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149047.041, "dur": 1.100,
    "args": {
      "External id": 2485680,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149049.963, "dur": 1.090,
    "args": {
      "External id": 2485681,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149052.945, "dur": 1.034,
    "args": {
      "External id": 2485682,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149055.815, "dur": 3.752,
    "args": {
      "External id": 2485683,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149061.001, "dur": 1.278,
    "args": {
      "External id": 2485684,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149064.068, "dur": 1.058,
    "args": {
      "External id": 2485685,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149067.020, "dur": 1.021,
    "args": {
      "External id": 2485686,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149087.605, "dur": 168.592,
    "args": {
      "External id": 2485687,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149117.896, "dur": 133.661,
    "args": {
      "External id": 2485688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149137.597, "dur": 12.286,
    "args": {
      "External id": 2485689,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149153.142, "dur": 69.826,
    "args": {
      "External id": 2485690,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149155.705, "dur": 66.903,
    "args": {
      "External id": 2485691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149160.079, "dur": 6.456,
    "args": {
      "External id": 2485692,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149168.434, "dur": 53.381,
    "args": {
      "External id": 2485693,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2492
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.20", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149381.861, "dur": 565.652,
    "args": {
      "External id": 2485694,"Record function id": 0, "Ev Idx": 2493
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149399.417, "dur": 535.816,
    "args": {
      "External id": 2485695,"Record function id": 0, "Ev Idx": 2494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149461.387, "dur": 4.839,
    "args": {
      "External id": 2485696,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149483.193, "dur": 31.620,
    "args": {
      "External id": 2485697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149488.368, "dur": 1.559,
    "args": {
      "External id": 2485698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149492.456, "dur": 2.140,
    "args": {
      "External id": 2485699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149496.607, "dur": 0.425,
    "args": {
      "External id": 2485700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149499.020, "dur": 0.401,
    "args": {
      "External id": 2485701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149501.359, "dur": 0.389,
    "args": {
      "External id": 2485702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149503.355, "dur": 0.489,
    "args": {
      "External id": 2485703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149505.552, "dur": 0.559,
    "args": {
      "External id": 2485704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149507.876, "dur": 0.407,
    "args": {
      "External id": 2485705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149510.105, "dur": 0.420,
    "args": {
      "External id": 2485706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149524.793, "dur": 49.788,
    "args": {
      "External id": 2485707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149610.773, "dur": 107.880,
    "args": {
      "External id": 2485708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149621.677, "dur": 5.615,
    "args": {
      "External id": 2485709,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149632.729, "dur": 10.990,
    "args": {
      "External id": 2485710,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149637.430, "dur": 5.868,
    "args": {
      "External id": 2485711,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149641.046, "dur": 0.845,
    "args": {
      "External id": 2485712,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149651.108, "dur": 26.850,
    "args": {
      "External id": 2485713,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149653.269, "dur": 0.422,
    "args": {
      "External id": 2485714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149655.558, "dur": 0.540,
    "args": {
      "External id": 2485715,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149658.008, "dur": 0.563,
    "args": {
      "External id": 2485716,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149660.312, "dur": 0.482,
    "args": {
      "External id": 2485717,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149662.524, "dur": 0.435,
    "args": {
      "External id": 2485718,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149664.749, "dur": 0.341,
    "args": {
      "External id": 2485719,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149666.888, "dur": 2.209,
    "args": {
      "External id": 2485720,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149670.739, "dur": 0.442,
    "args": {
      "External id": 2485721,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149672.936, "dur": 0.529,
    "args": {
      "External id": 2485722,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149690.224, "dur": 20.162,
    "args": {
      "External id": 2485723,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149764.302, "dur": 111.255,
    "args": {
      "External id": 2485724,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149786.311, "dur": 85.874,
    "args": {
      "External id": 2485725,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2524, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149796.569, "dur": 71.080,
    "args": {
      "External id": 2485726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149888.042, "dur": 1.707,
    "args": {
      "External id": 2485727,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2526, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657149963.373, "dur": 1706.737,
    "args": {
      "External id": 2485728,"Sequence number": 24740464, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2527
    }
  },
  {
    "ph": "f", "id": 184, "pid": 1336760, "tid": 1381209, "ts": 1514657149963.373,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150119.783, "dur": 106.017,
    "args": {
      "External id": 2485729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150262.756, "dur": 40.997,
    "args": {
      "External id": 2485730,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150323.706, "dur": 49.431,
    "args": {
      "External id": 2485731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150382.623, "dur": 31.934,
    "args": {
      "External id": 2485732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150421.419, "dur": 45.138,
    "args": {
      "External id": 2485733,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150476.431, "dur": 27.342,
    "args": {
      "External id": 2485734,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150511.752, "dur": 58.755,
    "args": {
      "External id": 2485735,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150596.263, "dur": 24.110,
    "args": {
      "External id": 2485736,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150639.324, "dur": 27.282,
    "args": {
      "External id": 2485737,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150687.132, "dur": 19.049,
    "args": {
      "External id": 2485738,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150720.767, "dur": 14.790,
    "args": {
      "External id": 2485739,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150748.556, "dur": 31.847,
    "args": {
      "External id": 2485740,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150784.260, "dur": 32.501,
    "args": {
      "External id": 2485741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150843.707, "dur": 216.258,
    "args": {
      "External id": 2485742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150923.681, "dur": 6.462,
    "args": {
      "External id": 2485743,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657150932.363, "dur": 4.543,
    "args": {
      "External id": 2485744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151120.898, "dur": 28.204,
    "args": {
      "External id": 2485745,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151162.706, "dur": 14.257,
    "args": {
      "External id": 2485746,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151187.020, "dur": 42.831,
    "args": {
      "External id": 2485747,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151236.773, "dur": 35.776,
    "args": {
      "External id": 2485748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151283.916, "dur": 21.401,
    "args": {
      "External id": 2485749,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151325.446, "dur": 43.163,
    "args": {
      "External id": 2485750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151378.177, "dur": 20.982,
    "args": {
      "External id": 2485751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151407.124, "dur": 31.960,
    "args": {
      "External id": 2485752,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151457.215, "dur": 22.586,
    "args": {
      "External id": 2485753,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151499.450, "dur": 23.373,
    "args": {
      "External id": 2485754,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151556.386, "dur": 18.788,
    "args": {
      "External id": 2485755,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151591.758, "dur": 18.847,
    "args": {
      "External id": 2485756,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151625.257, "dur": 15.732,
    "args": {
      "External id": 2485757,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151715.875, "dur": 41.826,
    "args": {
      "External id": 2485758,"Record function id": 0, "Ev Idx": 2557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151719.215, "dur": 37.286,
    "args": {
      "External id": 2485759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151723.630, "dur": 31.745,
    "args": {
      "External id": 2485760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151750.691, "dur": 4.550,
    "args": {
      "External id": 2485761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151761.950, "dur": 5.098,
    "args": {
      "External id": 2485762,"Record function id": 0, "Ev Idx": 2561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151763.399, "dur": 3.188,
    "args": {
      "External id": 2485763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151764.108, "dur": 1.830,
    "args": {
      "External id": 2485764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151764.764, "dur": 1.095,
    "args": {
      "External id": 2485765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151770.254, "dur": 4.454,
    "args": {
      "External id": 2485766,"Record function id": 0, "Ev Idx": 2565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151771.709, "dur": 2.598,
    "args": {
      "External id": 2485767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151772.328, "dur": 1.244,
    "args": {
      "External id": 2485768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151772.666, "dur": 0.831,
    "args": {
      "External id": 2485769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151777.881, "dur": 5.921,
    "args": {
      "External id": 2485770,"Record function id": 0, "Ev Idx": 2569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151779.070, "dur": 4.273,
    "args": {
      "External id": 2485771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151779.529, "dur": 3.015,
    "args": {
      "External id": 2485772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151779.934, "dur": 2.528,
    "args": {
      "External id": 2485773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151787.010, "dur": 4.092,
    "args": {
      "External id": 2485774,"Record function id": 0, "Ev Idx": 2573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151788.353, "dur": 2.333,
    "args": {
      "External id": 2485775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151788.848, "dur": 1.282,
    "args": {
      "External id": 2485776,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151789.333, "dur": 0.724,
    "args": {
      "External id": 2485777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151794.649, "dur": 3.968,
    "args": {
      "External id": 2485778,"Record function id": 0, "Ev Idx": 2577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151796.097, "dur": 2.117,
    "args": {
      "External id": 2485779,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151796.641, "dur": 1.173,
    "args": {
      "External id": 2485780,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151797.066, "dur": 0.682,
    "args": {
      "External id": 2485781,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151802.105, "dur": 3.647,
    "args": {
      "External id": 2485782,"Record function id": 0, "Ev Idx": 2581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151803.454, "dur": 1.905,
    "args": {
      "External id": 2485783,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151803.875, "dur": 1.047,
    "args": {
      "External id": 2485784,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151804.158, "dur": 0.685,
    "args": {
      "External id": 2485785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151809.103, "dur": 5.109,
    "args": {
      "External id": 2485786,"Record function id": 0, "Ev Idx": 2585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151810.168, "dur": 3.633,
    "args": {
      "External id": 2485787,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151810.721, "dur": 2.655,
    "args": {
      "External id": 2485788,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151812.624, "dur": 0.653,
    "args": {
      "External id": 2485789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151817.604, "dur": 3.795,
    "args": {
      "External id": 2485790,"Record function id": 0, "Ev Idx": 2589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151818.745, "dur": 2.252,
    "args": {
      "External id": 2485791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151819.182, "dur": 1.285,
    "args": {
      "External id": 2485792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151819.805, "dur": 0.578,
    "args": {
      "External id": 2485793,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151824.990, "dur": 39372.065,
    "args": {
      "External id": 2485794,"Record function id": 0, "Sequence number": 24740463, "Fwd thread id": 1, "Ev Idx": 2593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151826.450, "dur": 39362.197,
    "args": {
      "External id": 2485795,"Sequence number": 24740463, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2594
    }
  },
  {
    "ph": "f", "id": 185, "pid": 1336760, "tid": 1381209, "ts": 1514657151826.450,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151858.504, "dur": 38.417,
    "args": {
      "External id": 2485796,"Record function id": 0, "Ev Idx": 2595
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151904.381, "dur": 67.767,
    "args": {
      "External id": 2485797,"Record function id": 0, "Ev Idx": 2596
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.21)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657151977.576, "dur": 39199.545,
    "args": {
      "External id": 2485798,"Record function id": 0, "Ev Idx": 2597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152105.255, "dur": 7.655,
    "args": {
      "External id": 2485799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152123.822, "dur": 5.177,
    "args": {
      "External id": 2485800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152148.415, "dur": 38138.404,
    "args": {
      "External id": 2485801,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152161.748, "dur": 38116.280,
    "args": {
      "External id": 2485802,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152209.647, "dur": 16.564,
    "args": {
      "External id": 2485803,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152232.677, "dur": 38005.735,
    "args": {
      "External id": 2485804,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152235.511, "dur": 38002.199,
    "args": {
      "External id": 2485805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152239.595, "dur": 5.256,
    "args": {
      "External id": 2485806,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657152246.803, "dur": 37987.244,
    "args": {
      "External id": 2485807,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190378.221, "dur": 8.982,
    "args": {
      "External id": 2485808,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190381.326, "dur": 5.535,
    "args": {
      "External id": 2485809,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190417.191, "dur": 409.400,
    "args": {
      "External id": 2485810,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190443.970, "dur": 376.488,
    "args": {
      "External id": 2485811,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2610, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190456.270, "dur": 358.219,
    "args": {
      "External id": 2485812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190848.717, "dur": 2.332,
    "args": {
      "External id": 2485813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2612, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190915.562, "dur": 6.379,
    "args": {
      "External id": 2485814,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657190968.031, "dur": 1.547,
    "args": {
      "External id": 2485815,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191022.703, "dur": 1.891,
    "args": {
      "External id": 2485816,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191043.203, "dur": 0.917,
    "args": {
      "External id": 2485817,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191056.557, "dur": 0.964,
    "args": {
      "External id": 2485818,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191069.796, "dur": 1.082,
    "args": {
      "External id": 2485819,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191082.494, "dur": 1.078,
    "args": {
      "External id": 2485820,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191097.811, "dur": 2.421,
    "args": {
      "External id": 2485821,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191111.804, "dur": 0.989,
    "args": {
      "External id": 2485822,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191214.346, "dur": 2840.994,
    "args": {
      "External id": 2485823,"Record function id": 0, "Ev Idx": 2622
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191234.923, "dur": 1094.801,
    "args": {
      "External id": 2485824,"Record function id": 0, "Ev Idx": 2623
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191254.673, "dur": 360.899,
    "args": {
      "External id": 2485825,"Record function id": 0, "Ev Idx": 2624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191341.821, "dur": 4.312,
    "args": {
      "External id": 2485826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191349.508, "dur": 1.501,
    "args": {
      "External id": 2485827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191353.106, "dur": 1.034,
    "args": {
      "External id": 2485828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191355.675, "dur": 3.057,
    "args": {
      "External id": 2485829,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191360.192, "dur": 0.910,
    "args": {
      "External id": 2485830,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191362.688, "dur": 1.284,
    "args": {
      "External id": 2485831,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191365.468, "dur": 2.545,
    "args": {
      "External id": 2485832,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191369.575, "dur": 1.053,
    "args": {
      "External id": 2485833,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191372.231, "dur": 0.771,
    "args": {
      "External id": 2485834,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191374.539, "dur": 0.747,
    "args": {
      "External id": 2485835,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191395.125, "dur": 184.093,
    "args": {
      "External id": 2485836,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191410.912, "dur": 163.148,
    "args": {
      "External id": 2485837,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191436.335, "dur": 12.304,
    "args": {
      "External id": 2485838,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191451.981, "dur": 72.636,
    "args": {
      "External id": 2485839,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191455.056, "dur": 69.124,
    "args": {
      "External id": 2485840,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191459.025, "dur": 8.019,
    "args": {
      "External id": 2485841,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191469.030, "dur": 54.534,
    "args": {
      "External id": 2485842,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2641
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.19", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191710.257, "dur": 611.261,
    "args": {
      "External id": 2485843,"Record function id": 0, "Ev Idx": 2642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191728.133, "dur": 581.129,
    "args": {
      "External id": 2485844,"Record function id": 0, "Ev Idx": 2643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191792.378, "dur": 5.161,
    "args": {
      "External id": 2485845,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191814.268, "dur": 33.697,
    "args": {
      "External id": 2485846,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191819.509, "dur": 1.845,
    "args": {
      "External id": 2485847,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191823.640, "dur": 1.121,
    "args": {
      "External id": 2485848,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191826.922, "dur": 0.395,
    "args": {
      "External id": 2485849,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191828.934, "dur": 0.669,
    "args": {
      "External id": 2485850,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191831.505, "dur": 0.653,
    "args": {
      "External id": 2485851,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191834.025, "dur": 0.465,
    "args": {
      "External id": 2485852,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191836.107, "dur": 2.461,
    "args": {
      "External id": 2485853,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191840.344, "dur": 0.465,
    "args": {
      "External id": 2485854,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191842.278, "dur": 0.343,
    "args": {
      "External id": 2485855,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191858.469, "dur": 31.653,
    "args": {
      "External id": 2485856,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191922.078, "dur": 161.274,
    "args": {
      "External id": 2485857,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191932.390, "dur": 3.240,
    "args": {
      "External id": 2485858,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191941.227, "dur": 10.387,
    "args": {
      "External id": 2485859,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191945.616, "dur": 5.578,
    "args": {
      "External id": 2485860,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191949.374, "dur": 0.631,
    "args": {
      "External id": 2485861,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191958.609, "dur": 78.002,
    "args": {
      "External id": 2485862,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191960.816, "dur": 1.155,
    "args": {
      "External id": 2485863,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191964.293, "dur": 0.483,
    "args": {
      "External id": 2485864,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191966.537, "dur": 0.730,
    "args": {
      "External id": 2485865,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191969.360, "dur": 1.787,
    "args": {
      "External id": 2485866,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191972.789, "dur": 0.533,
    "args": {
      "External id": 2485867,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191975.292, "dur": 0.636,
    "args": {
      "External id": 2485868,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191977.864, "dur": 0.447,
    "args": {
      "External id": 2485869,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657191980.024, "dur": 0.444,
    "args": {
      "External id": 2485870,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192030.637, "dur": 0.544,
    "args": {
      "External id": 2485871,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192051.161, "dur": 23.596,
    "args": {
      "External id": 2485872,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192131.014, "dur": 114.516,
    "args": {
      "External id": 2485873,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192154.842, "dur": 87.356,
    "args": {
      "External id": 2485874,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2673, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192165.173, "dur": 72.137,
    "args": {
      "External id": 2485875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192259.054, "dur": 1.808,
    "args": {
      "External id": 2485876,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2675, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192337.631, "dur": 1692.874,
    "args": {
      "External id": 2485877,"Sequence number": 24740462, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2676
    }
  },
  {
    "ph": "f", "id": 186, "pid": 1336760, "tid": 1381209, "ts": 1514657192337.631,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192445.985, "dur": 123.761,
    "args": {
      "External id": 2485878,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192613.906, "dur": 43.523,
    "args": {
      "External id": 2485879,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192675.447, "dur": 54.580,
    "args": {
      "External id": 2485880,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192739.816, "dur": 34.029,
    "args": {
      "External id": 2485881,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192783.269, "dur": 45.612,
    "args": {
      "External id": 2485882,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192837.839, "dur": 28.430,
    "args": {
      "External id": 2485883,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192874.273, "dur": 43.566,
    "args": {
      "External id": 2485884,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657192941.354, "dur": 22.839,
    "args": {
      "External id": 2485885,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193022.804, "dur": 32.593,
    "args": {
      "External id": 2485886,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193078.954, "dur": 20.010,
    "args": {
      "External id": 2485887,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193112.899, "dur": 14.844,
    "args": {
      "External id": 2485888,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193138.744, "dur": 33.899,
    "args": {
      "External id": 2485889,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193176.172, "dur": 34.628,
    "args": {
      "External id": 2485890,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193240.122, "dur": 170.805,
    "args": {
      "External id": 2485891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193316.657, "dur": 6.202,
    "args": {
      "External id": 2485892,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193325.233, "dur": 2.509,
    "args": {
      "External id": 2485893,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193444.092, "dur": 25.814,
    "args": {
      "External id": 2485894,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193481.674, "dur": 14.406,
    "args": {
      "External id": 2485895,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193505.501, "dur": 56.762,
    "args": {
      "External id": 2485896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193571.788, "dur": 40.817,
    "args": {
      "External id": 2485897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193623.056, "dur": 38.543,
    "args": {
      "External id": 2485898,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193672.822, "dur": 38.742,
    "args": {
      "External id": 2485899,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193719.817, "dur": 24.298,
    "args": {
      "External id": 2485900,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193751.755, "dur": 29.791,
    "args": {
      "External id": 2485901,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193805.392, "dur": 24.435,
    "args": {
      "External id": 2485902,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193847.870, "dur": 28.326,
    "args": {
      "External id": 2485903,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193891.919, "dur": 16.094,
    "args": {
      "External id": 2485904,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193923.231, "dur": 14.263,
    "args": {
      "External id": 2485905,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657193949.905, "dur": 15.659,
    "args": {
      "External id": 2485906,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194078.692, "dur": 15.436,
    "args": {
      "External id": 2485907,"Record function id": 0, "Ev Idx": 2706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194082.246, "dur": 10.932,
    "args": {
      "External id": 2485908,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194086.488, "dur": 5.896,
    "args": {
      "External id": 2485909,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194088.114, "dur": 4.121,
    "args": {
      "External id": 2485910,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194098.770, "dur": 5.132,
    "args": {
      "External id": 2485911,"Record function id": 0, "Ev Idx": 2710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194099.968, "dur": 3.516,
    "args": {
      "External id": 2485912,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194100.791, "dur": 2.206,
    "args": {
      "External id": 2485913,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194101.855, "dur": 1.022,
    "args": {
      "External id": 2485914,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194107.626, "dur": 6.742,
    "args": {
      "External id": 2485915,"Record function id": 0, "Ev Idx": 2714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194108.875, "dur": 5.055,
    "args": {
      "External id": 2485916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194110.167, "dur": 3.311,
    "args": {
      "External id": 2485917,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194110.655, "dur": 2.729,
    "args": {
      "External id": 2485918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194117.878, "dur": 3.908,
    "args": {
      "External id": 2485919,"Record function id": 0, "Ev Idx": 2718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194118.930, "dur": 2.456,
    "args": {
      "External id": 2485920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194119.510, "dur": 1.476,
    "args": {
      "External id": 2485921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194119.986, "dur": 0.936,
    "args": {
      "External id": 2485922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194125.300, "dur": 3.622,
    "args": {
      "External id": 2485923,"Record function id": 0, "Ev Idx": 2722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194126.462, "dur": 2.060,
    "args": {
      "External id": 2485924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194127.057, "dur": 1.045,
    "args": {
      "External id": 2485925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194127.475, "dur": 0.561,
    "args": {
      "External id": 2485926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194132.420, "dur": 4.170,
    "args": {
      "External id": 2485927,"Record function id": 0, "Ev Idx": 2726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194133.425, "dur": 2.732,
    "args": {
      "External id": 2485928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194134.082, "dur": 1.560,
    "args": {
      "External id": 2485929,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194134.528, "dur": 1.005,
    "args": {
      "External id": 2485930,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194140.095, "dur": 3.960,
    "args": {
      "External id": 2485931,"Record function id": 0, "Ev Idx": 2730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194141.382, "dur": 2.277,
    "args": {
      "External id": 2485932,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194142.000, "dur": 1.244,
    "args": {
      "External id": 2485933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194142.462, "dur": 0.686,
    "args": {
      "External id": 2485934,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194147.483, "dur": 4.255,
    "args": {
      "External id": 2485935,"Record function id": 0, "Ev Idx": 2734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194148.742, "dur": 2.595,
    "args": {
      "External id": 2485936,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194149.695, "dur": 1.231,
    "args": {
      "External id": 2485937,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194150.189, "dur": 0.670,
    "args": {
      "External id": 2485938,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194155.187, "dur": 4.219,
    "args": {
      "External id": 2485939,"Record function id": 0, "Ev Idx": 2738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194156.235, "dur": 2.746,
    "args": {
      "External id": 2485940,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194157.079, "dur": 1.464,
    "args": {
      "External id": 2485941,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194157.761, "dur": 0.694,
    "args": {
      "External id": 2485942,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194163.383, "dur": 36611.176,
    "args": {
      "External id": 2485943,"Record function id": 0, "Sequence number": 24740461, "Fwd thread id": 1, "Ev Idx": 2742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194165.101, "dur": 36601.008,
    "args": {
      "External id": 2485944,"Sequence number": 24740461, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2743
    }
  },
  {
    "ph": "f", "id": 187, "pid": 1336760, "tid": 1381209, "ts": 1514657194165.101,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194197.657, "dur": 41.726,
    "args": {
      "External id": 2485945,"Record function id": 0, "Ev Idx": 2744
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194247.042, "dur": 70.251,
    "args": {
      "External id": 2485946,"Record function id": 0, "Ev Idx": 2745
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.20)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194323.029, "dur": 36435.231,
    "args": {
      "External id": 2485947,"Record function id": 0, "Ev Idx": 2746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194413.873, "dur": 6.480,
    "args": {
      "External id": 2485948,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194430.128, "dur": 6.458,
    "args": {
      "External id": 2485949,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194450.247, "dur": 35467.509,
    "args": {
      "External id": 2485950,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194464.788, "dur": 35443.239,
    "args": {
      "External id": 2485951,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194561.928, "dur": 12.975,
    "args": {
      "External id": 2485952,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194581.578, "dur": 35286.885,
    "args": {
      "External id": 2485953,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194584.388, "dur": 35283.435,
    "args": {
      "External id": 2485954,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194588.770, "dur": 6.089,
    "args": {
      "External id": 2485955,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657194596.934, "dur": 35267.325,
    "args": {
      "External id": 2485956,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230040.050, "dur": 9.759,
    "args": {
      "External id": 2485957,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230043.074, "dur": 6.233,
    "args": {
      "External id": 2485958,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230083.553, "dur": 360.875,
    "args": {
      "External id": 2485959,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230112.660, "dur": 326.478,
    "args": {
      "External id": 2485960,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2759, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230124.594, "dur": 309.388,
    "args": {
      "External id": 2485961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230463.120, "dur": 1.994,
    "args": {
      "External id": 2485962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2761, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230522.607, "dur": 21.791,
    "args": {
      "External id": 2485963,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230592.968, "dur": 1.984,
    "args": {
      "External id": 2485964,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230612.619, "dur": 2.959,
    "args": {
      "External id": 2485965,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230629.148, "dur": 1.074,
    "args": {
      "External id": 2485966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230642.136, "dur": 1.227,
    "args": {
      "External id": 2485967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230655.142, "dur": 0.981,
    "args": {
      "External id": 2485968,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230667.748, "dur": 2.822,
    "args": {
      "External id": 2485969,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230682.034, "dur": 2.265,
    "args": {
      "External id": 2485970,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230695.965, "dur": 0.738,
    "args": {
      "External id": 2485971,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230790.862, "dur": 2889.167,
    "args": {
      "External id": 2485972,"Record function id": 0, "Ev Idx": 2771
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230809.913, "dur": 1160.976,
    "args": {
      "External id": 2485973,"Record function id": 0, "Ev Idx": 2772
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230824.829, "dur": 380.657,
    "args": {
      "External id": 2485974,"Record function id": 0, "Ev Idx": 2773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230908.535, "dur": 4.109,
    "args": {
      "External id": 2485975,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230915.992, "dur": 0.993,
    "args": {
      "External id": 2485976,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230918.909, "dur": 3.949,
    "args": {
      "External id": 2485977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230924.838, "dur": 1.167,
    "args": {
      "External id": 2485978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230927.719, "dur": 0.812,
    "args": {
      "External id": 2485979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230937.072, "dur": 1.072,
    "args": {
      "External id": 2485980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230939.909, "dur": 2.550,
    "args": {
      "External id": 2485981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230943.954, "dur": 1.383,
    "args": {
      "External id": 2485982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230947.015, "dur": 0.864,
    "args": {
      "External id": 2485983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230949.299, "dur": 1.064,
    "args": {
      "External id": 2485984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657230969.169, "dur": 198.806,
    "args": {
      "External id": 2485985,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231024.497, "dur": 138.256,
    "args": {
      "External id": 2485986,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231045.419, "dur": 14.057,
    "args": {
      "External id": 2485987,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231062.838, "dur": 71.240,
    "args": {
      "External id": 2485988,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231068.574, "dur": 65.100,
    "args": {
      "External id": 2485989,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231072.738, "dur": 6.104,
    "args": {
      "External id": 2485990,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231080.762, "dur": 52.292,
    "args": {
      "External id": 2485991,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2790
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.18", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231301.770, "dur": 660.313,
    "args": {
      "External id": 2485992,"Record function id": 0, "Ev Idx": 2791
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231322.132, "dur": 626.886,
    "args": {
      "External id": 2485993,"Record function id": 0, "Ev Idx": 2792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231387.539, "dur": 4.854,
    "args": {
      "External id": 2485994,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231408.408, "dur": 33.619,
    "args": {
      "External id": 2485995,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231413.193, "dur": 1.741,
    "args": {
      "External id": 2485996,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231417.186, "dur": 0.444,
    "args": {
      "External id": 2485997,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231419.411, "dur": 0.567,
    "args": {
      "External id": 2485998,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231421.765, "dur": 0.542,
    "args": {
      "External id": 2485999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231424.240, "dur": 0.418,
    "args": {
      "External id": 2486000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231426.567, "dur": 2.763,
    "args": {
      "External id": 2486001,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231430.957, "dur": 0.583,
    "args": {
      "External id": 2486002,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231433.585, "dur": 0.520,
    "args": {
      "External id": 2486003,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231435.939, "dur": 0.374,
    "args": {
      "External id": 2486004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231452.379, "dur": 29.500,
    "args": {
      "External id": 2486005,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231513.286, "dur": 125.505,
    "args": {
      "External id": 2486006,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231523.121, "dur": 3.243,
    "args": {
      "External id": 2486007,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231549.023, "dur": 11.821,
    "args": {
      "External id": 2486008,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231553.672, "dur": 6.743,
    "args": {
      "External id": 2486009,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231557.708, "dur": 0.871,
    "args": {
      "External id": 2486010,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231569.508, "dur": 27.156,
    "args": {
      "External id": 2486011,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231572.059, "dur": 0.388,
    "args": {
      "External id": 2486012,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231574.388, "dur": 0.433,
    "args": {
      "External id": 2486013,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231576.342, "dur": 2.353,
    "args": {
      "External id": 2486014,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231580.736, "dur": 0.342,
    "args": {
      "External id": 2486015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231582.899, "dur": 0.601,
    "args": {
      "External id": 2486016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231585.304, "dur": 0.449,
    "args": {
      "External id": 2486017,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231587.243, "dur": 0.373,
    "args": {
      "External id": 2486018,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231589.309, "dur": 0.357,
    "args": {
      "External id": 2486019,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231591.126, "dur": 0.312,
    "args": {
      "External id": 2486020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231609.008, "dur": 22.273,
    "args": {
      "External id": 2486021,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231717.689, "dur": 161.243,
    "args": {
      "External id": 2486022,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231779.788, "dur": 95.561,
    "args": {
      "External id": 2486023,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2822, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231793.362, "dur": 77.714,
    "args": {
      "External id": 2486024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231893.324, "dur": 1.706,
    "args": {
      "External id": 2486025,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2824, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657231978.797, "dur": 1679.176,
    "args": {
      "External id": 2486026,"Sequence number": 24740460, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2825
    }
  },
  {
    "ph": "f", "id": 188, "pid": 1336760, "tid": 1381209, "ts": 1514657231978.797,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232130.509, "dur": 106.351,
    "args": {
      "External id": 2486027,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232275.205, "dur": 42.308,
    "args": {
      "External id": 2486028,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232336.751, "dur": 49.955,
    "args": {
      "External id": 2486029,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232398.163, "dur": 33.142,
    "args": {
      "External id": 2486030,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232438.571, "dur": 44.698,
    "args": {
      "External id": 2486031,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232491.527, "dur": 28.213,
    "args": {
      "External id": 2486032,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232543.533, "dur": 48.742,
    "args": {
      "External id": 2486033,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232621.061, "dur": 23.787,
    "args": {
      "External id": 2486034,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232664.161, "dur": 25.993,
    "args": {
      "External id": 2486035,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232710.823, "dur": 18.902,
    "args": {
      "External id": 2486036,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232743.088, "dur": 15.506,
    "args": {
      "External id": 2486037,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232768.910, "dur": 29.784,
    "args": {
      "External id": 2486038,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232802.285, "dur": 32.053,
    "args": {
      "External id": 2486039,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232862.589, "dur": 209.598,
    "args": {
      "External id": 2486040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232940.691, "dur": 6.003,
    "args": {
      "External id": 2486041,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657232948.781, "dur": 2.857,
    "args": {
      "External id": 2486042,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233109.561, "dur": 26.108,
    "args": {
      "External id": 2486043,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233147.534, "dur": 14.034,
    "args": {
      "External id": 2486044,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233170.961, "dur": 41.900,
    "args": {
      "External id": 2486045,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233231.849, "dur": 45.068,
    "args": {
      "External id": 2486046,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233288.628, "dur": 21.593,
    "args": {
      "External id": 2486047,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233316.230, "dur": 31.392,
    "args": {
      "External id": 2486048,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233358.046, "dur": 20.144,
    "args": {
      "External id": 2486049,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233386.610, "dur": 32.080,
    "args": {
      "External id": 2486050,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233440.686, "dur": 24.127,
    "args": {
      "External id": 2486051,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233482.787, "dur": 29.126,
    "args": {
      "External id": 2486052,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233541.613, "dur": 19.990,
    "args": {
      "External id": 2486053,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233583.622, "dur": 14.269,
    "args": {
      "External id": 2486054,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233611.091, "dur": 18.813,
    "args": {
      "External id": 2486055,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233706.584, "dur": 16.203,
    "args": {
      "External id": 2486056,"Record function id": 0, "Ev Idx": 2855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233710.155, "dur": 11.726,
    "args": {
      "External id": 2486057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233714.536, "dur": 6.280,
    "args": {
      "External id": 2486058,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233716.552, "dur": 4.095,
    "args": {
      "External id": 2486059,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233726.918, "dur": 6.888,
    "args": {
      "External id": 2486060,"Record function id": 0, "Ev Idx": 2859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233728.238, "dur": 5.103,
    "args": {
      "External id": 2486061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233729.174, "dur": 3.553,
    "args": {
      "External id": 2486062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233729.948, "dur": 2.676,
    "args": {
      "External id": 2486063,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233737.437, "dur": 4.974,
    "args": {
      "External id": 2486064,"Record function id": 0, "Ev Idx": 2863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233738.884, "dur": 3.105,
    "args": {
      "External id": 2486065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233739.881, "dur": 1.612,
    "args": {
      "External id": 2486066,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233740.406, "dur": 1.020,
    "args": {
      "External id": 2486067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233745.991, "dur": 5.031,
    "args": {
      "External id": 2486068,"Record function id": 0, "Ev Idx": 2867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233747.413, "dur": 3.214,
    "args": {
      "External id": 2486069,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233748.179, "dur": 1.915,
    "args": {
      "External id": 2486070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233749.045, "dur": 0.965,
    "args": {
      "External id": 2486071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233754.365, "dur": 4.862,
    "args": {
      "External id": 2486072,"Record function id": 0, "Ev Idx": 2871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233755.712, "dur": 3.116,
    "args": {
      "External id": 2486073,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233756.608, "dur": 1.798,
    "args": {
      "External id": 2486074,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233757.471, "dur": 0.847,
    "args": {
      "External id": 2486075,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233762.424, "dur": 4.330,
    "args": {
      "External id": 2486076,"Record function id": 0, "Ev Idx": 2875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233763.872, "dur": 2.492,
    "args": {
      "External id": 2486077,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233764.325, "dur": 1.552,
    "args": {
      "External id": 2486078,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233765.023, "dur": 0.751,
    "args": {
      "External id": 2486079,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233770.156, "dur": 4.605,
    "args": {
      "External id": 2486080,"Record function id": 0, "Ev Idx": 2879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233771.590, "dur": 2.747,
    "args": {
      "External id": 2486081,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233772.605, "dur": 1.293,
    "args": {
      "External id": 2486082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233773.163, "dur": 0.640,
    "args": {
      "External id": 2486083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233777.959, "dur": 4.122,
    "args": {
      "External id": 2486084,"Record function id": 0, "Ev Idx": 2883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233779.273, "dur": 2.410,
    "args": {
      "External id": 2486085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233779.931, "dur": 1.183,
    "args": {
      "External id": 2486086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233780.488, "dur": 0.558,
    "args": {
      "External id": 2486087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233785.290, "dur": 6.236,
    "args": {
      "External id": 2486088,"Record function id": 0, "Ev Idx": 2887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233786.661, "dur": 4.469,
    "args": {
      "External id": 2486089,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233787.399, "dur": 3.287,
    "args": {
      "External id": 2486090,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233787.970, "dur": 2.638,
    "args": {
      "External id": 2486091,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233795.324, "dur": 37994.992,
    "args": {
      "External id": 2486092,"Record function id": 0, "Sequence number": 24740459, "Fwd thread id": 1, "Ev Idx": 2891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233796.901, "dur": 37984.614,
    "args": {
      "External id": 2486093,"Sequence number": 24740459, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2892
    }
  },
  {
    "ph": "f", "id": 189, "pid": 1336760, "tid": 1381209, "ts": 1514657233796.901,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233823.716, "dur": 34.938,
    "args": {
      "External id": 2486094,"Record function id": 0, "Ev Idx": 2893
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233865.774, "dur": 63.604,
    "args": {
      "External id": 2486095,"Record function id": 0, "Ev Idx": 2894
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.19)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657233934.637, "dur": 37839.046,
    "args": {
      "External id": 2486096,"Record function id": 0, "Ev Idx": 2895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234059.477, "dur": 7.878,
    "args": {
      "External id": 2486097,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234078.075, "dur": 5.521,
    "args": {
      "External id": 2486098,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234101.955, "dur": 36804.785,
    "args": {
      "External id": 2486099,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234115.223, "dur": 36781.273,
    "args": {
      "External id": 2486100,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234200.990, "dur": 14.379,
    "args": {
      "External id": 2486101,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234221.845, "dur": 36633.393,
    "args": {
      "External id": 2486102,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234224.469, "dur": 36630.096,
    "args": {
      "External id": 2486103,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234228.710, "dur": 5.499,
    "args": {
      "External id": 2486104,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657234236.196, "dur": 36614.938,
    "args": {
      "External id": 2486105,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271034.167, "dur": 9.975,
    "args": {
      "External id": 2486106,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271037.600, "dur": 6.040,
    "args": {
      "External id": 2486107,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271073.822, "dur": 374.093,
    "args": {
      "External id": 2486108,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271101.874, "dur": 340.472,
    "args": {
      "External id": 2486109,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2908, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271113.805, "dur": 322.683,
    "args": {
      "External id": 2486110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271468.635, "dur": 2.518,
    "args": {
      "External id": 2486111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2910, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271546.408, "dur": 6.743,
    "args": {
      "External id": 2486112,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271604.688, "dur": 3.790,
    "args": {
      "External id": 2486113,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271626.250, "dur": 1.238,
    "args": {
      "External id": 2486114,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271639.601, "dur": 1.261,
    "args": {
      "External id": 2486115,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271652.643, "dur": 0.859,
    "args": {
      "External id": 2486116,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271669.000, "dur": 2.823,
    "args": {
      "External id": 2486117,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271683.834, "dur": 0.998,
    "args": {
      "External id": 2486118,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271697.864, "dur": 2.048,
    "args": {
      "External id": 2486119,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271712.069, "dur": 1.008,
    "args": {
      "External id": 2486120,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271806.333, "dur": 2794.889,
    "args": {
      "External id": 2486121,"Record function id": 0, "Ev Idx": 2920
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271825.669, "dur": 1080.500,
    "args": {
      "External id": 2486122,"Record function id": 0, "Ev Idx": 2921
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271840.368, "dur": 372.027,
    "args": {
      "External id": 2486123,"Record function id": 0, "Ev Idx": 2922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271921.711, "dur": 5.255,
    "args": {
      "External id": 2486124,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271929.969, "dur": 1.752,
    "args": {
      "External id": 2486125,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271934.033, "dur": 1.060,
    "args": {
      "External id": 2486126,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271936.969, "dur": 1.286,
    "args": {
      "External id": 2486127,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271940.067, "dur": 1.381,
    "args": {
      "External id": 2486128,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271943.350, "dur": 1.025,
    "args": {
      "External id": 2486129,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271946.373, "dur": 2.161,
    "args": {
      "External id": 2486130,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271950.239, "dur": 0.996,
    "args": {
      "External id": 2486131,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271952.664, "dur": 3.204,
    "args": {
      "External id": 2486132,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271957.295, "dur": 1.079,
    "args": {
      "External id": 2486133,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657271976.648, "dur": 200.384,
    "args": {
      "External id": 2486134,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272029.992, "dur": 142.134,
    "args": {
      "External id": 2486135,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272057.909, "dur": 12.938,
    "args": {
      "External id": 2486136,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272074.325, "dur": 69.253,
    "args": {
      "External id": 2486137,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272077.400, "dur": 65.769,
    "args": {
      "External id": 2486138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272081.676, "dur": 6.089,
    "args": {
      "External id": 2486139,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272089.879, "dur": 52.793,
    "args": {
      "External id": 2486140,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2939
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.17", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272303.723, "dur": 594.281,
    "args": {
      "External id": 2486141,"Record function id": 0, "Ev Idx": 2940
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272322.184, "dur": 563.031,
    "args": {
      "External id": 2486142,"Record function id": 0, "Ev Idx": 2941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272391.707, "dur": 4.271,
    "args": {
      "External id": 2486143,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272412.263, "dur": 37.521,
    "args": {
      "External id": 2486144,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272417.771, "dur": 1.791,
    "args": {
      "External id": 2486145,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272421.762, "dur": 0.627,
    "args": {
      "External id": 2486146,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272424.219, "dur": 0.426,
    "args": {
      "External id": 2486147,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272426.435, "dur": 2.410,
    "args": {
      "External id": 2486148,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272430.992, "dur": 0.414,
    "args": {
      "External id": 2486149,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272433.433, "dur": 0.357,
    "args": {
      "External id": 2486150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272440.112, "dur": 0.449,
    "args": {
      "External id": 2486151,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272442.161, "dur": 0.526,
    "args": {
      "External id": 2486152,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272444.487, "dur": 0.707,
    "args": {
      "External id": 2486153,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272459.559, "dur": 32.859,
    "args": {
      "External id": 2486154,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272526.834, "dur": 134.134,
    "args": {
      "External id": 2486155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272555.206, "dur": 4.814,
    "args": {
      "External id": 2486156,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272566.143, "dur": 11.114,
    "args": {
      "External id": 2486157,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272570.458, "dur": 6.373,
    "args": {
      "External id": 2486158,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272574.457, "dur": 0.831,
    "args": {
      "External id": 2486159,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272585.601, "dur": 29.827,
    "args": {
      "External id": 2486160,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272588.366, "dur": 2.884,
    "args": {
      "External id": 2486161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272593.010, "dur": 0.541,
    "args": {
      "External id": 2486162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272595.089, "dur": 0.550,
    "args": {
      "External id": 2486163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272597.542, "dur": 0.418,
    "args": {
      "External id": 2486164,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272599.611, "dur": 0.437,
    "args": {
      "External id": 2486165,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272601.571, "dur": 0.709,
    "args": {
      "External id": 2486166,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272604.139, "dur": 0.326,
    "args": {
      "External id": 2486167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272606.513, "dur": 0.546,
    "args": {
      "External id": 2486168,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272608.939, "dur": 2.207,
    "args": {
      "External id": 2486169,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272630.925, "dur": 22.326,
    "args": {
      "External id": 2486170,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272706.196, "dur": 113.648,
    "args": {
      "External id": 2486171,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272731.107, "dur": 85.555,
    "args": {
      "External id": 2486172,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2971, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272741.757, "dur": 70.507,
    "args": {
      "External id": 2486173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272836.743, "dur": 1.932,
    "args": {
      "External id": 2486174,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2973, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657272913.959, "dur": 1665.581,
    "args": {
      "External id": 2486175,"Sequence number": 24740458, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2974
    }
  },
  {
    "ph": "f", "id": 190, "pid": 1336760, "tid": 1381209, "ts": 1514657272913.959,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273070.657, "dur": 105.845,
    "args": {
      "External id": 2486176,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273213.982, "dur": 41.388,
    "args": {
      "External id": 2486177,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273275.620, "dur": 50.678,
    "args": {
      "External id": 2486178,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273336.057, "dur": 33.059,
    "args": {
      "External id": 2486179,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273376.995, "dur": 46.234,
    "args": {
      "External id": 2486180,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273431.751, "dur": 27.879,
    "args": {
      "External id": 2486181,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273469.976, "dur": 42.095,
    "args": {
      "External id": 2486182,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273552.395, "dur": 25.407,
    "args": {
      "External id": 2486183,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273596.669, "dur": 27.371,
    "args": {
      "External id": 2486184,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273644.307, "dur": 18.897,
    "args": {
      "External id": 2486185,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273675.896, "dur": 14.678,
    "args": {
      "External id": 2486186,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273702.806, "dur": 33.555,
    "args": {
      "External id": 2486187,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273740.028, "dur": 34.948,
    "args": {
      "External id": 2486188,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273801.029, "dur": 169.806,
    "args": {
      "External id": 2486189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273879.614, "dur": 5.963,
    "args": {
      "External id": 2486190,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657273887.984, "dur": 2.167,
    "args": {
      "External id": 2486191,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274045.066, "dur": 26.856,
    "args": {
      "External id": 2486192,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274083.753, "dur": 13.777,
    "args": {
      "External id": 2486193,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274107.399, "dur": 42.271,
    "args": {
      "External id": 2486194,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274158.199, "dur": 35.330,
    "args": {
      "External id": 2486195,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274200.897, "dur": 21.510,
    "args": {
      "External id": 2486196,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274228.504, "dur": 30.402,
    "args": {
      "External id": 2486197,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274280.650, "dur": 29.683,
    "args": {
      "External id": 2486198,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274321.529, "dur": 31.785,
    "args": {
      "External id": 2486199,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274371.494, "dur": 22.968,
    "args": {
      "External id": 2486200,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274410.989, "dur": 25.276,
    "args": {
      "External id": 2486201,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274453.788, "dur": 15.753,
    "args": {
      "External id": 2486202,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274483.520, "dur": 13.906,
    "args": {
      "External id": 2486203,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274513.528, "dur": 30.287,
    "args": {
      "External id": 2486204,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274627.745, "dur": 16.896,
    "args": {
      "External id": 2486205,"Record function id": 0, "Ev Idx": 3004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274631.141, "dur": 12.548,
    "args": {
      "External id": 2486206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274635.350, "dur": 7.240,
    "args": {
      "External id": 2486207,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274636.788, "dur": 5.674,
    "args": {
      "External id": 2486208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274648.871, "dur": 5.596,
    "args": {
      "External id": 2486209,"Record function id": 0, "Ev Idx": 3008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274650.509, "dur": 3.501,
    "args": {
      "External id": 2486210,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274651.429, "dur": 2.069,
    "args": {
      "External id": 2486211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274652.285, "dur": 1.065,
    "args": {
      "External id": 2486212,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274657.900, "dur": 4.432,
    "args": {
      "External id": 2486213,"Record function id": 0, "Ev Idx": 3012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274659.291, "dur": 2.623,
    "args": {
      "External id": 2486214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274660.013, "dur": 1.412,
    "args": {
      "External id": 2486215,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274660.460, "dur": 0.890,
    "args": {
      "External id": 2486216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274665.698, "dur": 5.368,
    "args": {
      "External id": 2486217,"Record function id": 0, "Ev Idx": 3016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274666.809, "dur": 3.858,
    "args": {
      "External id": 2486218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274667.452, "dur": 2.635,
    "args": {
      "External id": 2486219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274667.917, "dur": 2.094,
    "args": {
      "External id": 2486220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274674.258, "dur": 7.534,
    "args": {
      "External id": 2486221,"Record function id": 0, "Ev Idx": 3020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274679.010, "dur": 2.385,
    "args": {
      "External id": 2486222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274679.475, "dur": 1.440,
    "args": {
      "External id": 2486223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274680.132, "dur": 0.705,
    "args": {
      "External id": 2486224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274685.199, "dur": 4.173,
    "args": {
      "External id": 2486225,"Record function id": 0, "Ev Idx": 3024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274686.628, "dur": 2.358,
    "args": {
      "External id": 2486226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274687.287, "dur": 1.256,
    "args": {
      "External id": 2486227,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274687.803, "dur": 0.636,
    "args": {
      "External id": 2486228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274692.538, "dur": 4.244,
    "args": {
      "External id": 2486229,"Record function id": 0, "Ev Idx": 3028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274694.059, "dur": 2.336,
    "args": {
      "External id": 2486230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274694.766, "dur": 1.163,
    "args": {
      "External id": 2486231,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274695.059, "dur": 0.783,
    "args": {
      "External id": 2486232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274700.120, "dur": 5.564,
    "args": {
      "External id": 2486233,"Record function id": 0, "Ev Idx": 3032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274701.379, "dur": 3.902,
    "args": {
      "External id": 2486234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274702.032, "dur": 2.805,
    "args": {
      "External id": 2486235,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274702.312, "dur": 2.446,
    "args": {
      "External id": 2486236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274709.372, "dur": 4.363,
    "args": {
      "External id": 2486237,"Record function id": 0, "Ev Idx": 3036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274710.733, "dur": 2.574,
    "args": {
      "External id": 2486238,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274711.196, "dur": 1.692,
    "args": {
      "External id": 2486239,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274711.815, "dur": 0.983,
    "args": {
      "External id": 2486240,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274717.443, "dur": 36408.114,
    "args": {
      "External id": 2486241,"Record function id": 0, "Sequence number": 24740457, "Fwd thread id": 1, "Ev Idx": 3040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274718.963, "dur": 36397.826,
    "args": {
      "External id": 2486242,"Sequence number": 24740457, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3041
    }
  },
  {
    "ph": "f", "id": 191, "pid": 1336760, "tid": 1381209, "ts": 1514657274718.963,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274751.981, "dur": 37.594,
    "args": {
      "External id": 2486243,"Record function id": 0, "Ev Idx": 3042
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274797.163, "dur": 61.679,
    "args": {
      "External id": 2486244,"Record function id": 0, "Ev Idx": 3043
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.18)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274864.247, "dur": 36243.711,
    "args": {
      "External id": 2486245,"Record function id": 0, "Ev Idx": 3044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274951.684, "dur": 6.725,
    "args": {
      "External id": 2486246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657274968.377, "dur": 4.996,
    "args": {
      "External id": 2486247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275026.415, "dur": 35164.469,
    "args": {
      "External id": 2486248,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275073.526, "dur": 35107.144,
    "args": {
      "External id": 2486249,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275123.516, "dur": 14.492,
    "args": {
      "External id": 2486250,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275144.775, "dur": 34996.973,
    "args": {
      "External id": 2486251,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275147.566, "dur": 34993.517,
    "args": {
      "External id": 2486252,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275151.759, "dur": 6.359,
    "args": {
      "External id": 2486253,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657275160.017, "dur": 34977.711,
    "args": {
      "External id": 2486254,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310283.832, "dur": 8.953,
    "args": {
      "External id": 2486255,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310287.145, "dur": 5.295,
    "args": {
      "External id": 2486256,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310323.548, "dur": 418.763,
    "args": {
      "External id": 2486257,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310349.958, "dur": 386.518,
    "args": {
      "External id": 2486258,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3057, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310361.850, "dur": 368.278,
    "args": {
      "External id": 2486259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310764.842, "dur": 2.766,
    "args": {
      "External id": 2486260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3059, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310834.876, "dur": 6.817,
    "args": {
      "External id": 2486261,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310890.956, "dur": 1.382,
    "args": {
      "External id": 2486262,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310910.694, "dur": 1.227,
    "args": {
      "External id": 2486263,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310926.818, "dur": 1.142,
    "args": {
      "External id": 2486264,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310942.392, "dur": 1.094,
    "args": {
      "External id": 2486265,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310958.593, "dur": 0.922,
    "args": {
      "External id": 2486266,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657310972.135, "dur": 1.039,
    "args": {
      "External id": 2486267,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311022.760, "dur": 2.706,
    "args": {
      "External id": 2486268,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311040.165, "dur": 1.036,
    "args": {
      "External id": 2486269,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311142.804, "dur": 2897.286,
    "args": {
      "External id": 2486270,"Record function id": 0, "Ev Idx": 3069
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311162.653, "dur": 1122.939,
    "args": {
      "External id": 2486271,"Record function id": 0, "Ev Idx": 3070
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311179.188, "dur": 377.398,
    "args": {
      "External id": 2486272,"Record function id": 0, "Ev Idx": 3071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311299.973, "dur": 4.355,
    "args": {
      "External id": 2486273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311307.799, "dur": 1.175,
    "args": {
      "External id": 2486274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311311.045, "dur": 1.178,
    "args": {
      "External id": 2486275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311313.923, "dur": 1.056,
    "args": {
      "External id": 2486276,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311316.551, "dur": 1.231,
    "args": {
      "External id": 2486277,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311320.680, "dur": 0.947,
    "args": {
      "External id": 2486278,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311323.276, "dur": 2.161,
    "args": {
      "External id": 2486279,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311326.869, "dur": 2.978,
    "args": {
      "External id": 2486280,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311331.438, "dur": 1.167,
    "args": {
      "External id": 2486281,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311334.324, "dur": 0.766,
    "args": {
      "External id": 2486282,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311353.169, "dur": 150.535,
    "args": {
      "External id": 2486283,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311369.289, "dur": 129.561,
    "args": {
      "External id": 2486284,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311385.026, "dur": 13.477,
    "args": {
      "External id": 2486285,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311401.651, "dur": 68.962,
    "args": {
      "External id": 2486286,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311404.297, "dur": 65.975,
    "args": {
      "External id": 2486287,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311408.478, "dur": 5.996,
    "args": {
      "External id": 2486288,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311416.495, "dur": 52.905,
    "args": {
      "External id": 2486289,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3088
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.16", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311656.064, "dur": 619.839,
    "args": {
      "External id": 2486290,"Record function id": 0, "Ev Idx": 3089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311676.336, "dur": 587.100,
    "args": {
      "External id": 2486291,"Record function id": 0, "Ev Idx": 3090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311744.189, "dur": 9.337,
    "args": {
      "External id": 2486292,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311770.658, "dur": 30.331,
    "args": {
      "External id": 2486293,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311775.702, "dur": 1.567,
    "args": {
      "External id": 2486294,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311779.450, "dur": 0.388,
    "args": {
      "External id": 2486295,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311781.578, "dur": 2.556,
    "args": {
      "External id": 2486296,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311785.567, "dur": 0.840,
    "args": {
      "External id": 2486297,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311788.001, "dur": 0.432,
    "args": {
      "External id": 2486298,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311789.886, "dur": 0.481,
    "args": {
      "External id": 2486299,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311791.697, "dur": 0.434,
    "args": {
      "External id": 2486300,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311793.902, "dur": 0.473,
    "args": {
      "External id": 2486301,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311796.069, "dur": 0.409,
    "args": {
      "External id": 2486302,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311811.791, "dur": 33.021,
    "args": {
      "External id": 2486303,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311878.282, "dur": 149.320,
    "args": {
      "External id": 2486304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311889.121, "dur": 2.803,
    "args": {
      "External id": 2486305,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311897.218, "dur": 12.631,
    "args": {
      "External id": 2486306,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311901.652, "dur": 7.758,
    "args": {
      "External id": 2486307,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311905.379, "dur": 2.696,
    "args": {
      "External id": 2486308,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311917.353, "dur": 26.397,
    "args": {
      "External id": 2486309,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311919.448, "dur": 0.426,
    "args": {
      "External id": 2486310,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311922.109, "dur": 0.605,
    "args": {
      "External id": 2486311,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311924.320, "dur": 0.503,
    "args": {
      "External id": 2486312,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311926.587, "dur": 0.715,
    "args": {
      "External id": 2486313,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311928.731, "dur": 0.623,
    "args": {
      "External id": 2486314,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311931.245, "dur": 0.524,
    "args": {
      "External id": 2486315,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311933.314, "dur": 0.465,
    "args": {
      "External id": 2486316,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311935.697, "dur": 2.073,
    "args": {
      "External id": 2486317,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311939.432, "dur": 0.409,
    "args": {
      "External id": 2486318,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657311960.949, "dur": 55.835,
    "args": {
      "External id": 2486319,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312079.236, "dur": 117.036,
    "args": {
      "External id": 2486320,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312106.920, "dur": 86.033,
    "args": {
      "External id": 2486321,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3120, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312116.813, "dur": 72.126,
    "args": {
      "External id": 2486322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312212.624, "dur": 1.701,
    "args": {
      "External id": 2486323,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3122, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312293.447, "dur": 1667.452,
    "args": {
      "External id": 2486324,"Sequence number": 24740456, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3123
    }
  },
  {
    "ph": "f", "id": 192, "pid": 1336760, "tid": 1381209, "ts": 1514657312293.447,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312403.492, "dur": 103.419,
    "args": {
      "External id": 2486325,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312561.997, "dur": 44.650,
    "args": {
      "External id": 2486326,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312629.598, "dur": 54.809,
    "args": {
      "External id": 2486327,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312694.879, "dur": 33.913,
    "args": {
      "External id": 2486328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312735.902, "dur": 46.097,
    "args": {
      "External id": 2486329,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312789.756, "dur": 28.112,
    "args": {
      "External id": 2486330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312827.515, "dur": 41.774,
    "args": {
      "External id": 2486331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312891.998, "dur": 21.273,
    "args": {
      "External id": 2486332,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312932.229, "dur": 27.986,
    "args": {
      "External id": 2486333,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657312979.390, "dur": 55.410,
    "args": {
      "External id": 2486334,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313051.616, "dur": 16.496,
    "args": {
      "External id": 2486335,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313081.295, "dur": 34.082,
    "args": {
      "External id": 2486336,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313119.300, "dur": 33.672,
    "args": {
      "External id": 2486337,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313178.574, "dur": 172.722,
    "args": {
      "External id": 2486338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313260.728, "dur": 6.067,
    "args": {
      "External id": 2486339,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313268.784, "dur": 2.128,
    "args": {
      "External id": 2486340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313385.390, "dur": 26.476,
    "args": {
      "External id": 2486341,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313437.469, "dur": 14.538,
    "args": {
      "External id": 2486342,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313463.630, "dur": 38.177,
    "args": {
      "External id": 2486343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313509.001, "dur": 76.901,
    "args": {
      "External id": 2486344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313608.070, "dur": 28.933,
    "args": {
      "External id": 2486345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313642.980, "dur": 34.576,
    "args": {
      "External id": 2486346,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313689.858, "dur": 20.946,
    "args": {
      "External id": 2486347,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313718.431, "dur": 32.122,
    "args": {
      "External id": 2486348,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313770.293, "dur": 24.841,
    "args": {
      "External id": 2486349,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313812.816, "dur": 25.643,
    "args": {
      "External id": 2486350,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313853.336, "dur": 16.937,
    "args": {
      "External id": 2486351,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313887.373, "dur": 14.255,
    "args": {
      "External id": 2486352,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657313913.877, "dur": 14.503,
    "args": {
      "External id": 2486353,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314065.432, "dur": 17.233,
    "args": {
      "External id": 2486354,"Record function id": 0, "Ev Idx": 3153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314069.134, "dur": 12.380,
    "args": {
      "External id": 2486355,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314073.597, "dur": 6.237,
    "args": {
      "External id": 2486356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314075.222, "dur": 4.497,
    "args": {
      "External id": 2486357,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314086.854, "dur": 5.609,
    "args": {
      "External id": 2486358,"Record function id": 0, "Ev Idx": 3157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314088.367, "dur": 3.625,
    "args": {
      "External id": 2486359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314089.196, "dur": 2.249,
    "args": {
      "External id": 2486360,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314089.972, "dur": 1.331,
    "args": {
      "External id": 2486361,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314095.811, "dur": 4.203,
    "args": {
      "External id": 2486362,"Record function id": 0, "Ev Idx": 3161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314097.114, "dur": 2.492,
    "args": {
      "External id": 2486363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314097.611, "dur": 1.542,
    "args": {
      "External id": 2486364,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314098.018, "dur": 1.042,
    "args": {
      "External id": 2486365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314103.246, "dur": 4.990,
    "args": {
      "External id": 2486366,"Record function id": 0, "Ev Idx": 3165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314105.261, "dur": 2.581,
    "args": {
      "External id": 2486367,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314106.021, "dur": 1.381,
    "args": {
      "External id": 2486368,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314106.374, "dur": 0.966,
    "args": {
      "External id": 2486369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314111.427, "dur": 4.024,
    "args": {
      "External id": 2486370,"Record function id": 0, "Ev Idx": 3169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314112.650, "dur": 2.327,
    "args": {
      "External id": 2486371,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314113.281, "dur": 1.250,
    "args": {
      "External id": 2486372,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314113.617, "dur": 0.850,
    "args": {
      "External id": 2486373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314118.643, "dur": 3.934,
    "args": {
      "External id": 2486374,"Record function id": 0, "Ev Idx": 3173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314119.919, "dur": 2.216,
    "args": {
      "External id": 2486375,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314120.363, "dur": 1.292,
    "args": {
      "External id": 2486376,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314121.025, "dur": 0.524,
    "args": {
      "External id": 2486377,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314126.155, "dur": 5.841,
    "args": {
      "External id": 2486378,"Record function id": 0, "Ev Idx": 3177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314127.390, "dur": 4.157,
    "args": {
      "External id": 2486379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314128.008, "dur": 3.028,
    "args": {
      "External id": 2486380,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314128.434, "dur": 2.507,
    "args": {
      "External id": 2486381,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314135.290, "dur": 3.672,
    "args": {
      "External id": 2486382,"Record function id": 0, "Ev Idx": 3181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314136.378, "dur": 2.165,
    "args": {
      "External id": 2486383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314136.820, "dur": 1.311,
    "args": {
      "External id": 2486384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314137.249, "dur": 0.796,
    "args": {
      "External id": 2486385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314142.233, "dur": 4.447,
    "args": {
      "External id": 2486386,"Record function id": 0, "Ev Idx": 3185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314143.565, "dur": 2.706,
    "args": {
      "External id": 2486387,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314144.350, "dur": 1.483,
    "args": {
      "External id": 2486388,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314145.074, "dur": 0.669,
    "args": {
      "External id": 2486389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314150.805, "dur": 36316.736,
    "args": {
      "External id": 2486390,"Record function id": 0, "Sequence number": 24740455, "Fwd thread id": 1, "Ev Idx": 3189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314152.255, "dur": 36306.695,
    "args": {
      "External id": 2486391,"Sequence number": 24740455, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3190
    }
  },
  {
    "ph": "f", "id": 193, "pid": 1336760, "tid": 1381209, "ts": 1514657314152.255,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314185.136, "dur": 42.879,
    "args": {
      "External id": 2486392,"Record function id": 0, "Ev Idx": 3191
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314235.717, "dur": 70.089,
    "args": {
      "External id": 2486393,"Record function id": 0, "Ev Idx": 3192
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.17)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314311.328, "dur": 36139.451,
    "args": {
      "External id": 2486394,"Record function id": 0, "Ev Idx": 3193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314400.260, "dur": 7.010,
    "args": {
      "External id": 2486395,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314416.553, "dur": 4.945,
    "args": {
      "External id": 2486396,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314437.045, "dur": 35159.587,
    "args": {
      "External id": 2486397,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314450.282, "dur": 35135.797,
    "args": {
      "External id": 2486398,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314505.026, "dur": 13.333,
    "args": {
      "External id": 2486399,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314524.751, "dur": 35016.974,
    "args": {
      "External id": 2486400,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314544.262, "dur": 34996.404,
    "args": {
      "External id": 2486401,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314549.806, "dur": 6.447,
    "args": {
      "External id": 2486402,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657314558.433, "dur": 34966.599,
    "args": {
      "External id": 2486403,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657349695.369, "dur": 10.518,
    "args": {
      "External id": 2486404,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657349698.913, "dur": 6.599,
    "args": {
      "External id": 2486405,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657349738.521, "dur": 405.692,
    "args": {
      "External id": 2486406,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657349766.166, "dur": 372.424,
    "args": {
      "External id": 2486407,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3206, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657349778.505, "dur": 353.119,
    "args": {
      "External id": 2486408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350166.239, "dur": 2.199,
    "args": {
      "External id": 2486409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3208, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350231.538, "dur": 8.232,
    "args": {
      "External id": 2486410,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350284.912, "dur": 1.304,
    "args": {
      "External id": 2486411,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350303.857, "dur": 0.836,
    "args": {
      "External id": 2486412,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350318.211, "dur": 0.912,
    "args": {
      "External id": 2486413,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350330.795, "dur": 3.153,
    "args": {
      "External id": 2486414,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350346.001, "dur": 1.285,
    "args": {
      "External id": 2486415,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350359.065, "dur": 1.327,
    "args": {
      "External id": 2486416,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350374.202, "dur": 1.855,
    "args": {
      "External id": 2486417,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350387.988, "dur": 2.471,
    "args": {
      "External id": 2486418,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350483.387, "dur": 2849.133,
    "args": {
      "External id": 2486419,"Record function id": 0, "Ev Idx": 3218
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350502.666, "dur": 1109.014,
    "args": {
      "External id": 2486420,"Record function id": 0, "Ev Idx": 3219
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350517.579, "dur": 356.011,
    "args": {
      "External id": 2486421,"Record function id": 0, "Ev Idx": 3220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350624.732, "dur": 4.575,
    "args": {
      "External id": 2486422,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350632.781, "dur": 1.265,
    "args": {
      "External id": 2486423,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350635.866, "dur": 1.449,
    "args": {
      "External id": 2486424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350639.080, "dur": 1.149,
    "args": {
      "External id": 2486425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350641.794, "dur": 1.271,
    "args": {
      "External id": 2486426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350644.812, "dur": 1.076,
    "args": {
      "External id": 2486427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350647.442, "dur": 3.904,
    "args": {
      "External id": 2486428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350652.862, "dur": 0.932,
    "args": {
      "External id": 2486429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350655.493, "dur": 0.845,
    "args": {
      "External id": 2486430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350657.965, "dur": 1.226,
    "args": {
      "External id": 2486431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350677.697, "dur": 161.921,
    "args": {
      "External id": 2486432,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350694.237, "dur": 140.490,
    "args": {
      "External id": 2486433,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350719.344, "dur": 13.569,
    "args": {
      "External id": 2486434,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350736.404, "dur": 68.999,
    "args": {
      "External id": 2486435,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350739.041, "dur": 66.066,
    "args": {
      "External id": 2486436,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350743.325, "dur": 5.476,
    "args": {
      "External id": 2486437,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350750.789, "dur": 53.762,
    "args": {
      "External id": 2486438,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3237
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.15", "pid": 1336760, "tid": 1381209,
    "ts": 1514657350969.859, "dur": 633.846,
    "args": {
      "External id": 2486439,"Record function id": 0, "Ev Idx": 3238
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351030.980, "dur": 559.690,
    "args": {
      "External id": 2486440,"Record function id": 0, "Ev Idx": 3239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351101.082, "dur": 6.221,
    "args": {
      "External id": 2486441,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351123.915, "dur": 33.802,
    "args": {
      "External id": 2486442,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351129.460, "dur": 1.548,
    "args": {
      "External id": 2486443,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351133.295, "dur": 2.813,
    "args": {
      "External id": 2486444,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351138.034, "dur": 0.409,
    "args": {
      "External id": 2486445,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351140.023, "dur": 0.578,
    "args": {
      "External id": 2486446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351142.652, "dur": 0.325,
    "args": {
      "External id": 2486447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351144.907, "dur": 0.541,
    "args": {
      "External id": 2486448,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351147.526, "dur": 0.979,
    "args": {
      "External id": 2486449,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351150.120, "dur": 0.390,
    "args": {
      "External id": 2486450,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351152.130, "dur": 0.388,
    "args": {
      "External id": 2486451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351167.685, "dur": 34.583,
    "args": {
      "External id": 2486452,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351235.213, "dur": 105.232,
    "args": {
      "External id": 2486453,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351245.297, "dur": 4.692,
    "args": {
      "External id": 2486454,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351255.481, "dur": 9.896,
    "args": {
      "External id": 2486455,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351259.644, "dur": 5.324,
    "args": {
      "External id": 2486456,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351263.205, "dur": 0.610,
    "args": {
      "External id": 2486457,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351272.589, "dur": 28.367,
    "args": {
      "External id": 2486458,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351274.870, "dur": 0.416,
    "args": {
      "External id": 2486459,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351277.479, "dur": 0.595,
    "args": {
      "External id": 2486460,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351280.055, "dur": 0.498,
    "args": {
      "External id": 2486461,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351282.429, "dur": 0.369,
    "args": {
      "External id": 2486462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351284.962, "dur": 0.412,
    "args": {
      "External id": 2486463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351286.929, "dur": 0.354,
    "args": {
      "External id": 2486464,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351289.112, "dur": 2.431,
    "args": {
      "External id": 2486465,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351293.169, "dur": 0.380,
    "args": {
      "External id": 2486466,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351295.667, "dur": 0.395,
    "args": {
      "External id": 2486467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351313.507, "dur": 19.365,
    "args": {
      "External id": 2486468,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351386.108, "dur": 115.582,
    "args": {
      "External id": 2486469,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351408.774, "dur": 89.085,
    "args": {
      "External id": 2486470,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3269, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351419.149, "dur": 74.365,
    "args": {
      "External id": 2486471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351517.999, "dur": 2.046,
    "args": {
      "External id": 2486472,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3271, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351619.908, "dur": 1690.227,
    "args": {
      "External id": 2486473,"Sequence number": 24740454, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3272
    }
  },
  {
    "ph": "f", "id": 194, "pid": 1336760, "tid": 1381209, "ts": 1514657351619.908,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351730.219, "dur": 105.774,
    "args": {
      "External id": 2486474,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351872.281, "dur": 41.309,
    "args": {
      "External id": 2486475,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657351934.396, "dur": 91.212,
    "args": {
      "External id": 2486476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352040.311, "dur": 40.272,
    "args": {
      "External id": 2486477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352088.295, "dur": 45.897,
    "args": {
      "External id": 2486478,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352163.296, "dur": 29.892,
    "args": {
      "External id": 2486479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352201.879, "dur": 43.873,
    "args": {
      "External id": 2486480,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352270.926, "dur": 24.028,
    "args": {
      "External id": 2486481,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352313.197, "dur": 27.255,
    "args": {
      "External id": 2486482,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352361.312, "dur": 19.582,
    "args": {
      "External id": 2486483,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352395.050, "dur": 15.482,
    "args": {
      "External id": 2486484,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352422.079, "dur": 29.563,
    "args": {
      "External id": 2486485,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352455.014, "dur": 32.990,
    "args": {
      "External id": 2486486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352513.110, "dur": 199.934,
    "args": {
      "External id": 2486487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352614.941, "dur": 6.554,
    "args": {
      "External id": 2486488,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352623.941, "dur": 3.554,
    "args": {
      "External id": 2486489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352750.604, "dur": 24.410,
    "args": {
      "External id": 2486490,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352786.352, "dur": 13.612,
    "args": {
      "External id": 2486491,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352808.710, "dur": 37.907,
    "args": {
      "External id": 2486492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352852.764, "dur": 37.395,
    "args": {
      "External id": 2486493,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352896.757, "dur": 37.531,
    "args": {
      "External id": 2486494,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657352946.000, "dur": 34.939,
    "args": {
      "External id": 2486495,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353024.745, "dur": 25.217,
    "args": {
      "External id": 2486496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353058.247, "dur": 32.404,
    "args": {
      "External id": 2486497,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353111.923, "dur": 27.143,
    "args": {
      "External id": 2486498,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353158.222, "dur": 25.331,
    "args": {
      "External id": 2486499,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353199.896, "dur": 17.112,
    "args": {
      "External id": 2486500,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353233.870, "dur": 14.535,
    "args": {
      "External id": 2486501,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353265.500, "dur": 16.509,
    "args": {
      "External id": 2486502,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353355.687, "dur": 15.883,
    "args": {
      "External id": 2486503,"Record function id": 0, "Ev Idx": 3302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353359.020, "dur": 11.738,
    "args": {
      "External id": 2486504,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353363.192, "dur": 6.456,
    "args": {
      "External id": 2486505,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353365.290, "dur": 4.219,
    "args": {
      "External id": 2486506,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353376.042, "dur": 4.807,
    "args": {
      "External id": 2486507,"Record function id": 0, "Ev Idx": 3306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353377.785, "dur": 2.634,
    "args": {
      "External id": 2486508,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353378.534, "dur": 1.429,
    "args": {
      "External id": 2486509,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353379.085, "dur": 0.798,
    "args": {
      "External id": 2486510,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353384.549, "dur": 4.057,
    "args": {
      "External id": 2486511,"Record function id": 0, "Ev Idx": 3310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353385.830, "dur": 2.382,
    "args": {
      "External id": 2486512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353386.525, "dur": 1.146,
    "args": {
      "External id": 2486513,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353387.015, "dur": 0.590,
    "args": {
      "External id": 2486514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353392.128, "dur": 7.070,
    "args": {
      "External id": 2486515,"Record function id": 0, "Ev Idx": 3314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353393.594, "dur": 5.156,
    "args": {
      "External id": 2486516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353394.314, "dur": 3.877,
    "args": {
      "External id": 2486517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353395.057, "dur": 3.007,
    "args": {
      "External id": 2486518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353402.910, "dur": 4.741,
    "args": {
      "External id": 2486519,"Record function id": 0, "Ev Idx": 3318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353404.913, "dur": 2.328,
    "args": {
      "External id": 2486520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353405.740, "dur": 1.088,
    "args": {
      "External id": 2486521,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353406.043, "dur": 0.704,
    "args": {
      "External id": 2486522,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353411.200, "dur": 4.051,
    "args": {
      "External id": 2486523,"Record function id": 0, "Ev Idx": 3322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353412.330, "dur": 2.513,
    "args": {
      "External id": 2486524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353413.018, "dur": 1.209,
    "args": {
      "External id": 2486525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353413.463, "dur": 0.681,
    "args": {
      "External id": 2486526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353418.753, "dur": 4.012,
    "args": {
      "External id": 2486527,"Record function id": 0, "Ev Idx": 3326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353419.895, "dur": 2.467,
    "args": {
      "External id": 2486528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353420.390, "dur": 1.486,
    "args": {
      "External id": 2486529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353420.869, "dur": 0.902,
    "args": {
      "External id": 2486530,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353426.109, "dur": 3.957,
    "args": {
      "External id": 2486531,"Record function id": 0, "Ev Idx": 3330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353427.181, "dur": 2.480,
    "args": {
      "External id": 2486532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353427.793, "dur": 1.316,
    "args": {
      "External id": 2486533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353428.071, "dur": 0.973,
    "args": {
      "External id": 2486534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353433.689, "dur": 4.199,
    "args": {
      "External id": 2486535,"Record function id": 0, "Ev Idx": 3334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353435.152, "dur": 2.354,
    "args": {
      "External id": 2486536,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353435.670, "dur": 1.285,
    "args": {
      "External id": 2486537,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353436.175, "dur": 0.695,
    "args": {
      "External id": 2486538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353442.018, "dur": 37052.979,
    "args": {
      "External id": 2486539,"Record function id": 0, "Sequence number": 24740453, "Fwd thread id": 1, "Ev Idx": 3338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353443.756, "dur": 37042.887,
    "args": {
      "External id": 2486540,"Sequence number": 24740453, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3339
    }
  },
  {
    "ph": "f", "id": 195, "pid": 1336760, "tid": 1381209, "ts": 1514657353443.756,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353472.100, "dur": 33.168,
    "args": {
      "External id": 2486541,"Record function id": 0, "Ev Idx": 3340
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353512.275, "dur": 82.879,
    "args": {
      "External id": 2486542,"Record function id": 0, "Ev Idx": 3341
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.16)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353602.363, "dur": 36876.461,
    "args": {
      "External id": 2486543,"Record function id": 0, "Ev Idx": 3342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353688.540, "dur": 11.109,
    "args": {
      "External id": 2486544,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353709.821, "dur": 5.288,
    "args": {
      "External id": 2486545,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353728.695, "dur": 35855.767,
    "args": {
      "External id": 2486546,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353742.520, "dur": 35832.028,
    "args": {
      "External id": 2486547,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353799.664, "dur": 15.792,
    "args": {
      "External id": 2486548,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353821.856, "dur": 35702.606,
    "args": {
      "External id": 2486549,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353824.442, "dur": 35699.216,
    "args": {
      "External id": 2486550,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353828.949, "dur": 4.644,
    "args": {
      "External id": 2486551,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657353835.575, "dur": 35684.493,
    "args": {
      "External id": 2486552,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657389673.893, "dur": 8.757,
    "args": {
      "External id": 2486553,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657389676.868, "dur": 5.437,
    "args": {
      "External id": 2486554,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657389709.692, "dur": 446.284,
    "args": {
      "External id": 2486555,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657389739.320, "dur": 411.061,
    "args": {
      "External id": 2486556,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3355, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657389751.596, "dur": 392.263,
    "args": {
      "External id": 2486557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390187.039, "dur": 2.471,
    "args": {
      "External id": 2486558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3357, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390253.197, "dur": 6.890,
    "args": {
      "External id": 2486559,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390306.248, "dur": 1.438,
    "args": {
      "External id": 2486560,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390326.143, "dur": 1.252,
    "args": {
      "External id": 2486561,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390346.831, "dur": 1.240,
    "args": {
      "External id": 2486562,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390360.303, "dur": 0.941,
    "args": {
      "External id": 2486563,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390373.903, "dur": 0.803,
    "args": {
      "External id": 2486564,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390387.401, "dur": 1.015,
    "args": {
      "External id": 2486565,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390406.448, "dur": 2.073,
    "args": {
      "External id": 2486566,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390419.147, "dur": 1.001,
    "args": {
      "External id": 2486567,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390510.510, "dur": 2809.437,
    "args": {
      "External id": 2486568,"Record function id": 0, "Ev Idx": 3367
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390549.270, "dur": 1039.266,
    "args": {
      "External id": 2486569,"Record function id": 0, "Ev Idx": 3368
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390564.232, "dur": 326.343,
    "args": {
      "External id": 2486570,"Record function id": 0, "Ev Idx": 3369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390650.687, "dur": 4.451,
    "args": {
      "External id": 2486571,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390658.389, "dur": 1.162,
    "args": {
      "External id": 2486572,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390661.616, "dur": 1.273,
    "args": {
      "External id": 2486573,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390664.533, "dur": 2.982,
    "args": {
      "External id": 2486574,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390669.245, "dur": 1.195,
    "args": {
      "External id": 2486575,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390672.191, "dur": 1.084,
    "args": {
      "External id": 2486576,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390674.757, "dur": 1.888,
    "args": {
      "External id": 2486577,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390678.028, "dur": 1.242,
    "args": {
      "External id": 2486578,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390681.131, "dur": 0.919,
    "args": {
      "External id": 2486579,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390683.616, "dur": 0.968,
    "args": {
      "External id": 2486580,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390702.113, "dur": 156.153,
    "args": {
      "External id": 2486581,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390718.713, "dur": 134.832,
    "args": {
      "External id": 2486582,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390740.897, "dur": 12.028,
    "args": {
      "External id": 2486583,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390756.146, "dur": 68.466,
    "args": {
      "External id": 2486584,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390758.679, "dur": 65.584,
    "args": {
      "External id": 2486585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390763.201, "dur": 7.251,
    "args": {
      "External id": 2486586,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390772.341, "dur": 50.986,
    "args": {
      "External id": 2486587,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3386
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.14", "pid": 1336760, "tid": 1381209,
    "ts": 1514657390979.380, "dur": 600.591,
    "args": {
      "External id": 2486588,"Record function id": 0, "Ev Idx": 3387
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391037.586, "dur": 529.696,
    "args": {
      "External id": 2486589,"Record function id": 0, "Ev Idx": 3388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391102.550, "dur": 5.926,
    "args": {
      "External id": 2486590,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391125.181, "dur": 32.320,
    "args": {
      "External id": 2486591,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391130.583, "dur": 1.938,
    "args": {
      "External id": 2486592,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391134.613, "dur": 0.340,
    "args": {
      "External id": 2486593,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391136.663, "dur": 0.451,
    "args": {
      "External id": 2486594,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391139.043, "dur": 0.449,
    "args": {
      "External id": 2486595,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391141.333, "dur": 0.379,
    "args": {
      "External id": 2486596,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391143.919, "dur": 0.340,
    "args": {
      "External id": 2486597,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391146.159, "dur": 2.122,
    "args": {
      "External id": 2486598,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391150.125, "dur": 0.470,
    "args": {
      "External id": 2486599,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391152.226, "dur": 0.671,
    "args": {
      "External id": 2486600,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391167.717, "dur": 34.117,
    "args": {
      "External id": 2486601,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391234.054, "dur": 100.630,
    "args": {
      "External id": 2486602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391244.282, "dur": 2.539,
    "args": {
      "External id": 2486603,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391252.127, "dur": 10.526,
    "args": {
      "External id": 2486604,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391256.729, "dur": 5.526,
    "args": {
      "External id": 2486605,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391260.509, "dur": 0.617,
    "args": {
      "External id": 2486606,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391269.669, "dur": 27.356,
    "args": {
      "External id": 2486607,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391272.229, "dur": 0.653,
    "args": {
      "External id": 2486608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391274.942, "dur": 0.687,
    "args": {
      "External id": 2486609,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391277.666, "dur": 0.353,
    "args": {
      "External id": 2486610,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391280.050, "dur": 2.286,
    "args": {
      "External id": 2486611,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391284.243, "dur": 0.374,
    "args": {
      "External id": 2486612,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391286.372, "dur": 0.561,
    "args": {
      "External id": 2486613,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391288.620, "dur": 0.464,
    "args": {
      "External id": 2486614,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391291.042, "dur": 0.464,
    "args": {
      "External id": 2486615,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391293.074, "dur": 0.407,
    "args": {
      "External id": 2486616,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391307.805, "dur": 19.337,
    "args": {
      "External id": 2486617,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391379.449, "dur": 109.276,
    "args": {
      "External id": 2486618,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391400.646, "dur": 84.341,
    "args": {
      "External id": 2486619,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3418, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391410.097, "dur": 70.645,
    "args": {
      "External id": 2486620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391501.363, "dur": 2.018,
    "args": {
      "External id": 2486621,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3420, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391597.082, "dur": 1701.722,
    "args": {
      "External id": 2486622,"Sequence number": 24740452, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3421
    }
  },
  {
    "ph": "f", "id": 196, "pid": 1336760, "tid": 1381209, "ts": 1514657391597.082,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391708.756, "dur": 105.611,
    "args": {
      "External id": 2486623,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391853.727, "dur": 42.021,
    "args": {
      "External id": 2486624,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391913.841, "dur": 55.486,
    "args": {
      "External id": 2486625,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657391979.857, "dur": 83.624,
    "args": {
      "External id": 2486626,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392076.720, "dur": 46.668,
    "args": {
      "External id": 2486627,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392132.015, "dur": 27.701,
    "args": {
      "External id": 2486628,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392167.477, "dur": 41.822,
    "args": {
      "External id": 2486629,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392234.332, "dur": 24.322,
    "args": {
      "External id": 2486630,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392279.345, "dur": 27.756,
    "args": {
      "External id": 2486631,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392326.085, "dur": 17.660,
    "args": {
      "External id": 2486632,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392358.178, "dur": 14.097,
    "args": {
      "External id": 2486633,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392380.119, "dur": 27.694,
    "args": {
      "External id": 2486634,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392410.940, "dur": 32.353,
    "args": {
      "External id": 2486635,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392488.124, "dur": 194.916,
    "args": {
      "External id": 2486636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392586.110, "dur": 5.923,
    "args": {
      "External id": 2486637,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392594.308, "dur": 2.393,
    "args": {
      "External id": 2486638,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392716.832, "dur": 25.214,
    "args": {
      "External id": 2486639,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392753.176, "dur": 14.344,
    "args": {
      "External id": 2486640,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392776.238, "dur": 39.645,
    "args": {
      "External id": 2486641,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392823.146, "dur": 38.876,
    "args": {
      "External id": 2486642,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392870.789, "dur": 24.828,
    "args": {
      "External id": 2486643,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392901.207, "dur": 30.157,
    "args": {
      "External id": 2486644,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657392949.386, "dur": 34.015,
    "args": {
      "External id": 2486645,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393033.698, "dur": 39.418,
    "args": {
      "External id": 2486646,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393096.986, "dur": 25.741,
    "args": {
      "External id": 2486647,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393144.592, "dur": 24.253,
    "args": {
      "External id": 2486648,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393184.752, "dur": 20.566,
    "args": {
      "External id": 2486649,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393222.425, "dur": 15.192,
    "args": {
      "External id": 2486650,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393250.211, "dur": 19.510,
    "args": {
      "External id": 2486651,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393344.057, "dur": 16.148,
    "args": {
      "External id": 2486652,"Record function id": 0, "Ev Idx": 3451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393347.667, "dur": 11.486,
    "args": {
      "External id": 2486653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393352.044, "dur": 6.187,
    "args": {
      "External id": 2486654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393353.747, "dur": 4.369,
    "args": {
      "External id": 2486655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393364.164, "dur": 5.398,
    "args": {
      "External id": 2486656,"Record function id": 0, "Ev Idx": 3455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393365.934, "dur": 3.182,
    "args": {
      "External id": 2486657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393366.596, "dur": 2.036,
    "args": {
      "External id": 2486658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393367.585, "dur": 0.934,
    "args": {
      "External id": 2486659,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393372.923, "dur": 6.439,
    "args": {
      "External id": 2486660,"Record function id": 0, "Ev Idx": 3459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393374.233, "dur": 4.742,
    "args": {
      "External id": 2486661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393375.245, "dur": 3.296,
    "args": {
      "External id": 2486662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393375.827, "dur": 2.651,
    "args": {
      "External id": 2486663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393382.642, "dur": 4.844,
    "args": {
      "External id": 2486664,"Record function id": 0, "Ev Idx": 3463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393384.268, "dur": 2.810,
    "args": {
      "External id": 2486665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393384.975, "dur": 1.549,
    "args": {
      "External id": 2486666,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393385.537, "dur": 0.924,
    "args": {
      "External id": 2486667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393390.615, "dur": 4.542,
    "args": {
      "External id": 2486668,"Record function id": 0, "Ev Idx": 3467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393392.108, "dur": 2.674,
    "args": {
      "External id": 2486669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393392.569, "dur": 1.571,
    "args": {
      "External id": 2486670,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393393.171, "dur": 0.904,
    "args": {
      "External id": 2486671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393398.346, "dur": 4.112,
    "args": {
      "External id": 2486672,"Record function id": 0, "Ev Idx": 3471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393399.569, "dur": 2.486,
    "args": {
      "External id": 2486673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393400.381, "dur": 1.111,
    "args": {
      "External id": 2486674,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393400.809, "dur": 0.579,
    "args": {
      "External id": 2486675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393405.806, "dur": 3.567,
    "args": {
      "External id": 2486676,"Record function id": 0, "Ev Idx": 3475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393406.820, "dur": 2.151,
    "args": {
      "External id": 2486677,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393407.387, "dur": 0.974,
    "args": {
      "External id": 2486678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393407.695, "dur": 0.599,
    "args": {
      "External id": 2486679,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393412.631, "dur": 3.339,
    "args": {
      "External id": 2486680,"Record function id": 0, "Ev Idx": 3479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393413.708, "dur": 1.856,
    "args": {
      "External id": 2486681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393414.215, "dur": 0.952,
    "args": {
      "External id": 2486682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393414.491, "dur": 0.566,
    "args": {
      "External id": 2486683,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393419.213, "dur": 3.745,
    "args": {
      "External id": 2486684,"Record function id": 0, "Ev Idx": 3483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393420.401, "dur": 2.160,
    "args": {
      "External id": 2486685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393420.858, "dur": 1.119,
    "args": {
      "External id": 2486686,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393421.271, "dur": 0.594,
    "args": {
      "External id": 2486687,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393426.647, "dur": 36091.204,
    "args": {
      "External id": 2486688,"Record function id": 0, "Sequence number": 24740451, "Fwd thread id": 1, "Ev Idx": 3487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393428.331, "dur": 36080.837,
    "args": {
      "External id": 2486689,"Sequence number": 24740451, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3488
    }
  },
  {
    "ph": "f", "id": 197, "pid": 1336760, "tid": 1381209, "ts": 1514657393428.331,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393457.479, "dur": 36.845,
    "args": {
      "External id": 2486690,"Record function id": 0, "Ev Idx": 3489
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393502.150, "dur": 80.695,
    "args": {
      "External id": 2486691,"Record function id": 0, "Ev Idx": 3490
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.15)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393590.391, "dur": 35911.473,
    "args": {
      "External id": 2486692,"Record function id": 0, "Ev Idx": 3491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393678.600, "dur": 6.970,
    "args": {
      "External id": 2486693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393696.011, "dur": 7.256,
    "args": {
      "External id": 2486694,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393718.106, "dur": 34953.057,
    "args": {
      "External id": 2486695,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393731.913, "dur": 34929.563,
    "args": {
      "External id": 2486696,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393786.530, "dur": 13.913,
    "args": {
      "External id": 2486697,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393807.252, "dur": 34815.138,
    "args": {
      "External id": 2486698,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393810.293, "dur": 34811.297,
    "args": {
      "External id": 2486699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393814.292, "dur": 5.400,
    "args": {
      "External id": 2486700,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657393821.557, "dur": 34796.450,
    "args": {
      "External id": 2486701,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657428765.201, "dur": 8.883,
    "args": {
      "External id": 2486702,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657428768.375, "dur": 5.375,
    "args": {
      "External id": 2486703,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657428803.606, "dur": 394.116,
    "args": {
      "External id": 2486704,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657428830.944, "dur": 361.590,
    "args": {
      "External id": 2486705,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3504, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657428842.981, "dur": 343.158,
    "args": {
      "External id": 2486706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429218.478, "dur": 2.178,
    "args": {
      "External id": 2486707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3506, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429286.058, "dur": 9.836,
    "args": {
      "External id": 2486708,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429341.525, "dur": 1.360,
    "args": {
      "External id": 2486709,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429359.821, "dur": 3.184,
    "args": {
      "External id": 2486710,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429376.610, "dur": 1.348,
    "args": {
      "External id": 2486711,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429389.361, "dur": 1.024,
    "args": {
      "External id": 2486712,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429401.292, "dur": 0.941,
    "args": {
      "External id": 2486713,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429413.565, "dur": 3.144,
    "args": {
      "External id": 2486714,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429428.050, "dur": 1.993,
    "args": {
      "External id": 2486715,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429440.634, "dur": 0.975,
    "args": {
      "External id": 2486716,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429551.537, "dur": 2904.822,
    "args": {
      "External id": 2486717,"Record function id": 0, "Ev Idx": 3516
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429574.653, "dur": 1092.208,
    "args": {
      "External id": 2486718,"Record function id": 0, "Ev Idx": 3517
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429590.573, "dur": 329.966,
    "args": {
      "External id": 2486719,"Record function id": 0, "Ev Idx": 3518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429678.560, "dur": 4.817,
    "args": {
      "External id": 2486720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429686.535, "dur": 1.226,
    "args": {
      "External id": 2486721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429689.572, "dur": 2.804,
    "args": {
      "External id": 2486722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429693.843, "dur": 1.080,
    "args": {
      "External id": 2486723,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429696.376, "dur": 0.863,
    "args": {
      "External id": 2486724,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429698.774, "dur": 1.206,
    "args": {
      "External id": 2486725,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429701.454, "dur": 1.907,
    "args": {
      "External id": 2486726,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429704.794, "dur": 1.062,
    "args": {
      "External id": 2486727,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429707.694, "dur": 1.026,
    "args": {
      "External id": 2486728,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429710.267, "dur": 1.221,
    "args": {
      "External id": 2486729,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429729.004, "dur": 158.559,
    "args": {
      "External id": 2486730,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429744.712, "dur": 138.192,
    "args": {
      "External id": 2486731,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429768.880, "dur": 13.767,
    "args": {
      "External id": 2486732,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429786.219, "dur": 68.840,
    "args": {
      "External id": 2486733,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429788.943, "dur": 65.677,
    "args": {
      "External id": 2486734,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429793.076, "dur": 5.580,
    "args": {
      "External id": 2486735,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657429800.636, "dur": 53.356,
    "args": {
      "External id": 2486736,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3535
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.13", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430068.816, "dur": 590.681,
    "args": {
      "External id": 2486737,"Record function id": 0, "Ev Idx": 3536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430087.945, "dur": 559.148,
    "args": {
      "External id": 2486738,"Record function id": 0, "Ev Idx": 3537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430156.809, "dur": 6.118,
    "args": {
      "External id": 2486739,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430179.562, "dur": 32.789,
    "args": {
      "External id": 2486740,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430185.230, "dur": 1.607,
    "args": {
      "External id": 2486741,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430189.319, "dur": 0.515,
    "args": {
      "External id": 2486742,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430191.631, "dur": 0.565,
    "args": {
      "External id": 2486743,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430194.077, "dur": 0.351,
    "args": {
      "External id": 2486744,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430196.226, "dur": 0.389,
    "args": {
      "External id": 2486745,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430198.483, "dur": 2.400,
    "args": {
      "External id": 2486746,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430202.324, "dur": 0.459,
    "args": {
      "External id": 2486747,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430204.327, "dur": 0.532,
    "args": {
      "External id": 2486748,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430206.847, "dur": 0.535,
    "args": {
      "External id": 2486749,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430223.147, "dur": 35.200,
    "args": {
      "External id": 2486750,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430291.031, "dur": 109.182,
    "args": {
      "External id": 2486751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430301.017, "dur": 2.995,
    "args": {
      "External id": 2486752,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430314.580, "dur": 10.415,
    "args": {
      "External id": 2486753,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430318.997, "dur": 5.574,
    "args": {
      "External id": 2486754,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430322.697, "dur": 0.592,
    "args": {
      "External id": 2486755,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430334.650, "dur": 25.763,
    "args": {
      "External id": 2486756,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430336.882, "dur": 0.634,
    "args": {
      "External id": 2486757,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430339.252, "dur": 0.550,
    "args": {
      "External id": 2486758,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430341.336, "dur": 2.083,
    "args": {
      "External id": 2486759,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430345.081, "dur": 0.455,
    "args": {
      "External id": 2486760,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430346.974, "dur": 0.588,
    "args": {
      "External id": 2486761,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430349.170, "dur": 0.502,
    "args": {
      "External id": 2486762,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430351.746, "dur": 0.394,
    "args": {
      "External id": 2486763,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430353.721, "dur": 0.744,
    "args": {
      "External id": 2486764,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430356.352, "dur": 0.263,
    "args": {
      "External id": 2486765,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430373.154, "dur": 19.670,
    "args": {
      "External id": 2486766,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430445.246, "dur": 133.027,
    "args": {
      "External id": 2486767,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430466.755, "dur": 107.674,
    "args": {
      "External id": 2486768,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3567, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430477.138, "dur": 92.366,
    "args": {
      "External id": 2486769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430593.481, "dur": 2.051,
    "args": {
      "External id": 2486770,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3569, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430674.760, "dur": 1760.720,
    "args": {
      "External id": 2486771,"Sequence number": 24740450, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3570
    }
  },
  {
    "ph": "f", "id": 198, "pid": 1336760, "tid": 1381209, "ts": 1514657430674.760,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430786.929, "dur": 108.249,
    "args": {
      "External id": 2486772,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657430934.780, "dur": 40.204,
    "args": {
      "External id": 2486773,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431034.937, "dur": 58.862,
    "args": {
      "External id": 2486774,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431105.650, "dur": 34.462,
    "args": {
      "External id": 2486775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431147.234, "dur": 44.950,
    "args": {
      "External id": 2486776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431199.041, "dur": 26.581,
    "args": {
      "External id": 2486777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431232.755, "dur": 42.164,
    "args": {
      "External id": 2486778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431301.806, "dur": 23.432,
    "args": {
      "External id": 2486779,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431343.580, "dur": 34.392,
    "args": {
      "External id": 2486780,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431398.564, "dur": 19.679,
    "args": {
      "External id": 2486781,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431435.497, "dur": 14.916,
    "args": {
      "External id": 2486782,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431459.882, "dur": 29.050,
    "args": {
      "External id": 2486783,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431492.554, "dur": 33.649,
    "args": {
      "External id": 2486784,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431600.270, "dur": 176.023,
    "args": {
      "External id": 2486785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431678.419, "dur": 6.119,
    "args": {
      "External id": 2486786,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431686.843, "dur": 1.938,
    "args": {
      "External id": 2486787,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431811.752, "dur": 26.514,
    "args": {
      "External id": 2486788,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431849.656, "dur": 15.191,
    "args": {
      "External id": 2486789,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431876.983, "dur": 44.802,
    "args": {
      "External id": 2486790,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431928.896, "dur": 36.745,
    "args": {
      "External id": 2486791,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657431974.426, "dur": 79.466,
    "args": {
      "External id": 2486792,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432070.579, "dur": 37.920,
    "args": {
      "External id": 2486793,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432117.903, "dur": 24.044,
    "args": {
      "External id": 2486794,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432150.299, "dur": 32.793,
    "args": {
      "External id": 2486795,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432206.583, "dur": 27.027,
    "args": {
      "External id": 2486796,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432263.462, "dur": 37.627,
    "args": {
      "External id": 2486797,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432321.186, "dur": 18.860,
    "args": {
      "External id": 2486798,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432357.643, "dur": 15.796,
    "args": {
      "External id": 2486799,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432387.951, "dur": 15.646,
    "args": {
      "External id": 2486800,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432482.630, "dur": 15.011,
    "args": {
      "External id": 2486801,"Record function id": 0, "Ev Idx": 3600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432485.778, "dur": 11.013,
    "args": {
      "External id": 2486802,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432489.701, "dur": 6.098,
    "args": {
      "External id": 2486803,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432491.616, "dur": 4.026,
    "args": {
      "External id": 2486804,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432501.765, "dur": 7.679,
    "args": {
      "External id": 2486805,"Record function id": 0, "Ev Idx": 3604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432503.263, "dur": 5.747,
    "args": {
      "External id": 2486806,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432503.993, "dur": 4.357,
    "args": {
      "External id": 2486807,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432505.162, "dur": 3.128,
    "args": {
      "External id": 2486808,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432512.788, "dur": 4.966,
    "args": {
      "External id": 2486809,"Record function id": 0, "Ev Idx": 3608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432514.259, "dur": 3.077,
    "args": {
      "External id": 2486810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432515.036, "dur": 1.730,
    "args": {
      "External id": 2486811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432515.548, "dur": 1.139,
    "args": {
      "External id": 2486812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432521.161, "dur": 4.680,
    "args": {
      "External id": 2486813,"Record function id": 0, "Ev Idx": 3612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432522.539, "dur": 2.894,
    "args": {
      "External id": 2486814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432523.272, "dur": 1.584,
    "args": {
      "External id": 2486815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432523.950, "dur": 0.824,
    "args": {
      "External id": 2486816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432550.007, "dur": 6.808,
    "args": {
      "External id": 2486817,"Record function id": 0, "Ev Idx": 3616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432551.921, "dur": 4.201,
    "args": {
      "External id": 2486818,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432553.135, "dur": 2.187,
    "args": {
      "External id": 2486819,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432553.756, "dur": 1.398,
    "args": {
      "External id": 2486820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432560.689, "dur": 4.821,
    "args": {
      "External id": 2486821,"Record function id": 0, "Ev Idx": 3620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432562.212, "dur": 2.888,
    "args": {
      "External id": 2486822,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432562.685, "dur": 1.728,
    "args": {
      "External id": 2486823,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432563.620, "dur": 0.693,
    "args": {
      "External id": 2486824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432568.753, "dur": 4.556,
    "args": {
      "External id": 2486825,"Record function id": 0, "Ev Idx": 3624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432570.329, "dur": 2.557,
    "args": {
      "External id": 2486826,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432571.283, "dur": 1.164,
    "args": {
      "External id": 2486827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432571.789, "dur": 0.577,
    "args": {
      "External id": 2486828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432576.614, "dur": 3.856,
    "args": {
      "External id": 2486829,"Record function id": 0, "Ev Idx": 3628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432577.956, "dur": 2.110,
    "args": {
      "External id": 2486830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432578.543, "dur": 0.990,
    "args": {
      "External id": 2486831,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432578.797, "dur": 0.657,
    "args": {
      "External id": 2486832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432583.585, "dur": 5.932,
    "args": {
      "External id": 2486833,"Record function id": 0, "Ev Idx": 3632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432584.826, "dur": 4.263,
    "args": {
      "External id": 2486834,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432585.403, "dur": 3.279,
    "args": {
      "External id": 2486835,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432585.992, "dur": 2.607,
    "args": {
      "External id": 2486836,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432593.373, "dur": 35590.504,
    "args": {
      "External id": 2486837,"Record function id": 0, "Sequence number": 24740449, "Fwd thread id": 1, "Ev Idx": 3636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432594.962, "dur": 35580.512,
    "args": {
      "External id": 2486838,"Sequence number": 24740449, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3637
    }
  },
  {
    "ph": "f", "id": 199, "pid": 1336760, "tid": 1381209, "ts": 1514657432594.962,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432627.878, "dur": 37.360,
    "args": {
      "External id": 2486839,"Record function id": 0, "Ev Idx": 3638
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432673.103, "dur": 64.575,
    "args": {
      "External id": 2486840,"Record function id": 0, "Ev Idx": 3639
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.14)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432742.812, "dur": 35424.915,
    "args": {
      "External id": 2486841,"Record function id": 0, "Ev Idx": 3640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432831.416, "dur": 6.634,
    "args": {
      "External id": 2486842,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432847.830, "dur": 4.881,
    "args": {
      "External id": 2486843,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432867.056, "dur": 34455.206,
    "args": {
      "External id": 2486844,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432880.577, "dur": 34432.890,
    "args": {
      "External id": 2486845,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432930.755, "dur": 13.944,
    "args": {
      "External id": 2486846,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432951.037, "dur": 34324.127,
    "args": {
      "External id": 2486847,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432953.707, "dur": 34320.760,
    "args": {
      "External id": 2486848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432957.902, "dur": 5.548,
    "args": {
      "External id": 2486849,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657432965.325, "dur": 34305.661,
    "args": {
      "External id": 2486850,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467414.438, "dur": 8.752,
    "args": {
      "External id": 2486851,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467417.903, "dur": 4.958,
    "args": {
      "External id": 2486852,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467452.555, "dur": 377.889,
    "args": {
      "External id": 2486853,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467478.701, "dur": 346.124,
    "args": {
      "External id": 2486854,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3653, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467491.026, "dur": 327.110,
    "args": {
      "External id": 2486855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467851.748, "dur": 2.067,
    "args": {
      "External id": 2486856,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3655, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467912.986, "dur": 6.548,
    "args": {
      "External id": 2486857,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657467964.898, "dur": 3.724,
    "args": {
      "External id": 2486858,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468020.595, "dur": 1.707,
    "args": {
      "External id": 2486859,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468037.640, "dur": 1.021,
    "args": {
      "External id": 2486860,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468050.004, "dur": 0.914,
    "args": {
      "External id": 2486861,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468062.899, "dur": 2.224,
    "args": {
      "External id": 2486862,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468076.874, "dur": 0.818,
    "args": {
      "External id": 2486863,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468089.067, "dur": 1.909,
    "args": {
      "External id": 2486864,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468101.199, "dur": 1.032,
    "args": {
      "External id": 2486865,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468199.942, "dur": 2825.084,
    "args": {
      "External id": 2486866,"Record function id": 0, "Ev Idx": 3665
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468219.453, "dur": 1066.097,
    "args": {
      "External id": 2486867,"Record function id": 0, "Ev Idx": 3666
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468236.278, "dur": 347.523,
    "args": {
      "External id": 2486868,"Record function id": 0, "Ev Idx": 3667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468322.683, "dur": 6.172,
    "args": {
      "External id": 2486869,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468332.252, "dur": 0.985,
    "args": {
      "External id": 2486870,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468335.036, "dur": 0.949,
    "args": {
      "External id": 2486871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468337.756, "dur": 1.462,
    "args": {
      "External id": 2486872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468340.655, "dur": 1.357,
    "args": {
      "External id": 2486873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468343.471, "dur": 0.856,
    "args": {
      "External id": 2486874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468345.813, "dur": 1.789,
    "args": {
      "External id": 2486875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468349.350, "dur": 1.127,
    "args": {
      "External id": 2486876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468352.298, "dur": 2.811,
    "args": {
      "External id": 2486877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468356.731, "dur": 0.970,
    "args": {
      "External id": 2486878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468375.704, "dur": 171.797,
    "args": {
      "External id": 2486879,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468391.243, "dur": 135.562,
    "args": {
      "External id": 2486880,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468414.568, "dur": 12.085,
    "args": {
      "External id": 2486881,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468429.843, "dur": 68.105,
    "args": {
      "External id": 2486882,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468432.590, "dur": 64.984,
    "args": {
      "External id": 2486883,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468437.037, "dur": 5.870,
    "args": {
      "External id": 2486884,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468444.858, "dur": 52.146,
    "args": {
      "External id": 2486885,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3684
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.12", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468682.442, "dur": 594.739,
    "args": {
      "External id": 2486886,"Record function id": 0, "Ev Idx": 3685
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468700.776, "dur": 562.894,
    "args": {
      "External id": 2486887,"Record function id": 0, "Ev Idx": 3686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468767.452, "dur": 5.766,
    "args": {
      "External id": 2486888,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468789.892, "dur": 30.417,
    "args": {
      "External id": 2486889,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468794.784, "dur": 1.845,
    "args": {
      "External id": 2486890,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468798.718, "dur": 0.765,
    "args": {
      "External id": 2486891,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468801.230, "dur": 0.386,
    "args": {
      "External id": 2486892,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468803.433, "dur": 1.994,
    "args": {
      "External id": 2486893,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468806.841, "dur": 0.385,
    "args": {
      "External id": 2486894,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468808.811, "dur": 0.531,
    "args": {
      "External id": 2486895,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468810.933, "dur": 0.312,
    "args": {
      "External id": 2486896,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468812.781, "dur": 0.396,
    "args": {
      "External id": 2486897,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468814.601, "dur": 0.589,
    "args": {
      "External id": 2486898,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468830.622, "dur": 33.745,
    "args": {
      "External id": 2486899,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468896.472, "dur": 140.680,
    "args": {
      "External id": 2486900,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468906.975, "dur": 4.205,
    "args": {
      "External id": 2486901,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468916.463, "dur": 9.922,
    "args": {
      "External id": 2486902,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468920.711, "dur": 5.243,
    "args": {
      "External id": 2486903,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468924.257, "dur": 0.546,
    "args": {
      "External id": 2486904,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468933.525, "dur": 26.047,
    "args": {
      "External id": 2486905,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468936.068, "dur": 1.853,
    "args": {
      "External id": 2486906,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468939.548, "dur": 0.608,
    "args": {
      "External id": 2486907,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468941.869, "dur": 0.594,
    "args": {
      "External id": 2486908,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468943.985, "dur": 0.426,
    "args": {
      "External id": 2486909,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468946.226, "dur": 0.456,
    "args": {
      "External id": 2486910,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468948.272, "dur": 0.380,
    "args": {
      "External id": 2486911,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468950.255, "dur": 0.345,
    "args": {
      "External id": 2486912,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468952.161, "dur": 0.522,
    "args": {
      "External id": 2486913,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468954.465, "dur": 1.871,
    "args": {
      "External id": 2486914,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657468969.330, "dur": 58.027,
    "args": {
      "External id": 2486915,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469085.131, "dur": 114.092,
    "args": {
      "External id": 2486916,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469107.711, "dur": 88.204,
    "args": {
      "External id": 2486917,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3716, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469117.710, "dur": 73.909,
    "args": {
      "External id": 2486918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469212.193, "dur": 1.990,
    "args": {
      "External id": 2486919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3718, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469293.650, "dur": 1669.523,
    "args": {
      "External id": 2486920,"Sequence number": 24740448, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3719
    }
  },
  {
    "ph": "f", "id": 200, "pid": 1336760, "tid": 1381209, "ts": 1514657469293.650,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469407.024, "dur": 103.578,
    "args": {
      "External id": 2486921,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469567.614, "dur": 45.140,
    "args": {
      "External id": 2486922,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469633.975, "dur": 53.312,
    "args": {
      "External id": 2486923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469696.963, "dur": 33.726,
    "args": {
      "External id": 2486924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469738.333, "dur": 46.052,
    "args": {
      "External id": 2486925,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469792.994, "dur": 30.471,
    "args": {
      "External id": 2486926,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469833.182, "dur": 43.112,
    "args": {
      "External id": 2486927,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469898.435, "dur": 22.905,
    "args": {
      "External id": 2486928,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657469938.650, "dur": 27.687,
    "args": {
      "External id": 2486929,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470024.669, "dur": 22.849,
    "args": {
      "External id": 2486930,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470062.981, "dur": 14.768,
    "args": {
      "External id": 2486931,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470088.618, "dur": 35.621,
    "args": {
      "External id": 2486932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470128.153, "dur": 33.568,
    "args": {
      "External id": 2486933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470188.426, "dur": 167.760,
    "args": {
      "External id": 2486934,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470262.869, "dur": 6.253,
    "args": {
      "External id": 2486935,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470271.475, "dur": 2.388,
    "args": {
      "External id": 2486936,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470389.241, "dur": 24.146,
    "args": {
      "External id": 2486937,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470443.918, "dur": 15.617,
    "args": {
      "External id": 2486938,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470471.683, "dur": 34.626,
    "args": {
      "External id": 2486939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470515.779, "dur": 58.123,
    "args": {
      "External id": 2486940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470583.951, "dur": 23.613,
    "args": {
      "External id": 2486941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470612.975, "dur": 30.049,
    "args": {
      "External id": 2486942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470649.363, "dur": 41.550,
    "args": {
      "External id": 2486943,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470705.791, "dur": 37.239,
    "args": {
      "External id": 2486944,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470765.878, "dur": 24.240,
    "args": {
      "External id": 2486945,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470811.510, "dur": 25.126,
    "args": {
      "External id": 2486946,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470851.088, "dur": 19.394,
    "args": {
      "External id": 2486947,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470886.898, "dur": 15.382,
    "args": {
      "External id": 2486948,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657470916.346, "dur": 19.147,
    "args": {
      "External id": 2486949,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471053.509, "dur": 17.305,
    "args": {
      "External id": 2486950,"Record function id": 0, "Ev Idx": 3749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471056.871, "dur": 12.841,
    "args": {
      "External id": 2486951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471060.918, "dur": 7.480,
    "args": {
      "External id": 2486952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471062.483, "dur": 5.791,
    "args": {
      "External id": 2486953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471074.650, "dur": 6.173,
    "args": {
      "External id": 2486954,"Record function id": 0, "Ev Idx": 3753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471076.390, "dur": 3.960,
    "args": {
      "External id": 2486955,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471077.754, "dur": 2.122,
    "args": {
      "External id": 2486956,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471078.730, "dur": 1.043,
    "args": {
      "External id": 2486957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471084.205, "dur": 4.781,
    "args": {
      "External id": 2486958,"Record function id": 0, "Ev Idx": 3757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471085.497, "dur": 3.089,
    "args": {
      "External id": 2486959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471086.405, "dur": 1.709,
    "args": {
      "External id": 2486960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471086.823, "dur": 1.208,
    "args": {
      "External id": 2486961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471092.221, "dur": 7.993,
    "args": {
      "External id": 2486962,"Record function id": 0, "Ev Idx": 3761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471093.815, "dur": 2.562,
    "args": {
      "External id": 2486963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471094.523, "dur": 1.298,
    "args": {
      "External id": 2486964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471095.170, "dur": 0.586,
    "args": {
      "External id": 2486965,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471103.650, "dur": 4.375,
    "args": {
      "External id": 2486966,"Record function id": 0, "Ev Idx": 3765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471104.932, "dur": 2.673,
    "args": {
      "External id": 2486967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471105.611, "dur": 1.603,
    "args": {
      "External id": 2486968,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471106.157, "dur": 0.988,
    "args": {
      "External id": 2486969,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471111.190, "dur": 3.713,
    "args": {
      "External id": 2486970,"Record function id": 0, "Ev Idx": 3769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471112.352, "dur": 2.147,
    "args": {
      "External id": 2486971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471112.985, "dur": 1.102,
    "args": {
      "External id": 2486972,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471113.449, "dur": 0.527,
    "args": {
      "External id": 2486973,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471118.169, "dur": 3.662,
    "args": {
      "External id": 2486974,"Record function id": 0, "Ev Idx": 3773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471119.465, "dur": 1.974,
    "args": {
      "External id": 2486975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471120.071, "dur": 0.976,
    "args": {
      "External id": 2486976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471120.348, "dur": 0.603,
    "args": {
      "External id": 2486977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471125.058, "dur": 4.799,
    "args": {
      "External id": 2486978,"Record function id": 0, "Ev Idx": 3777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471126.065, "dur": 3.397,
    "args": {
      "External id": 2486979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471126.495, "dur": 2.361,
    "args": {
      "External id": 2486980,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471126.749, "dur": 2.044,
    "args": {
      "External id": 2486981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471132.902, "dur": 3.715,
    "args": {
      "External id": 2486982,"Record function id": 0, "Ev Idx": 3781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471134.097, "dur": 2.120,
    "args": {
      "External id": 2486983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471134.626, "dur": 1.209,
    "args": {
      "External id": 2486984,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471135.121, "dur": 0.626,
    "args": {
      "External id": 2486985,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471140.502, "dur": 38347.744,
    "args": {
      "External id": 2486986,"Record function id": 0, "Sequence number": 24740447, "Fwd thread id": 1, "Ev Idx": 3785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471141.935, "dur": 38337.095,
    "args": {
      "External id": 2486987,"Sequence number": 24740447, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3786
    }
  },
  {
    "ph": "f", "id": 201, "pid": 1336760, "tid": 1381209, "ts": 1514657471141.935,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471173.829, "dur": 36.637,
    "args": {
      "External id": 2486988,"Record function id": 0, "Ev Idx": 3787
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471217.883, "dur": 64.787,
    "args": {
      "External id": 2486989,"Record function id": 0, "Ev Idx": 3788
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.13)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471288.555, "dur": 38181.815,
    "args": {
      "External id": 2486990,"Record function id": 0, "Ev Idx": 3789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471376.415, "dur": 6.581,
    "args": {
      "External id": 2486991,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471392.365, "dur": 4.837,
    "args": {
      "External id": 2486992,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471410.855, "dur": 37186.898,
    "args": {
      "External id": 2486993,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471423.203, "dur": 37165.388,
    "args": {
      "External id": 2486994,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471470.117, "dur": 12.821,
    "args": {
      "External id": 2486995,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471489.304, "dur": 37058.880,
    "args": {
      "External id": 2486996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471491.912, "dur": 37055.568,
    "args": {
      "External id": 2486997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471496.165, "dur": 5.410,
    "args": {
      "External id": 2486998,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657471503.533, "dur": 37039.980,
    "args": {
      "External id": 2486999,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657508687.981, "dur": 8.878,
    "args": {
      "External id": 2487000,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657508691.349, "dur": 5.231,
    "args": {
      "External id": 2487001,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657508726.778, "dur": 436.922,
    "args": {
      "External id": 2487002,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657508753.754, "dur": 404.072,
    "args": {
      "External id": 2487003,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3802, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657508765.373, "dur": 386.544,
    "args": {
      "External id": 2487004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509186.322, "dur": 2.202,
    "args": {
      "External id": 2487005,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3804, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509250.125, "dur": 6.675,
    "args": {
      "External id": 2487006,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509305.031, "dur": 1.263,
    "args": {
      "External id": 2487007,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509322.206, "dur": 1.171,
    "args": {
      "External id": 2487008,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509336.754, "dur": 0.960,
    "args": {
      "External id": 2487009,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509350.877, "dur": 1.054,
    "args": {
      "External id": 2487010,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509365.909, "dur": 0.889,
    "args": {
      "External id": 2487011,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509379.769, "dur": 1.226,
    "args": {
      "External id": 2487012,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509393.551, "dur": 1.734,
    "args": {
      "External id": 2487013,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509405.896, "dur": 0.902,
    "args": {
      "External id": 2487014,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509503.938, "dur": 2773.598,
    "args": {
      "External id": 2487015,"Record function id": 0, "Ev Idx": 3814
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509526.806, "dur": 1057.315,
    "args": {
      "External id": 2487016,"Record function id": 0, "Ev Idx": 3815
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509557.830, "dur": 323.726,
    "args": {
      "External id": 2487017,"Record function id": 0, "Ev Idx": 3816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509643.826, "dur": 5.018,
    "args": {
      "External id": 2487018,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509652.285, "dur": 0.993,
    "args": {
      "External id": 2487019,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509655.379, "dur": 1.233,
    "args": {
      "External id": 2487020,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509658.099, "dur": 0.764,
    "args": {
      "External id": 2487021,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509660.619, "dur": 1.100,
    "args": {
      "External id": 2487022,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509663.504, "dur": 0.898,
    "args": {
      "External id": 2487023,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509666.154, "dur": 1.867,
    "args": {
      "External id": 2487024,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509669.387, "dur": 2.872,
    "args": {
      "External id": 2487025,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509674.092, "dur": 1.431,
    "args": {
      "External id": 2487026,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509677.231, "dur": 0.955,
    "args": {
      "External id": 2487027,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509695.283, "dur": 154.410,
    "args": {
      "External id": 2487028,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509710.624, "dur": 134.357,
    "args": {
      "External id": 2487029,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509733.682, "dur": 12.124,
    "args": {
      "External id": 2487030,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509748.992, "dur": 67.317,
    "args": {
      "External id": 2487031,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509751.622, "dur": 64.322,
    "args": {
      "External id": 2487032,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509755.535, "dur": 5.718,
    "args": {
      "External id": 2487033,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509763.141, "dur": 52.110,
    "args": {
      "External id": 2487034,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3833
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.11", "pid": 1336760, "tid": 1381209,
    "ts": 1514657509970.507, "dur": 605.603,
    "args": {
      "External id": 2487035,"Record function id": 0, "Ev Idx": 3834
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510024.909, "dur": 537.959,
    "args": {
      "External id": 2487036,"Record function id": 0, "Ev Idx": 3835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510098.568, "dur": 5.854,
    "args": {
      "External id": 2487037,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510121.289, "dur": 32.477,
    "args": {
      "External id": 2487038,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510126.674, "dur": 1.732,
    "args": {
      "External id": 2487039,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510130.289, "dur": 0.528,
    "args": {
      "External id": 2487040,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510133.199, "dur": 2.443,
    "args": {
      "External id": 2487041,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510137.730, "dur": 0.593,
    "args": {
      "External id": 2487042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510139.906, "dur": 0.426,
    "args": {
      "External id": 2487043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510142.835, "dur": 0.378,
    "args": {
      "External id": 2487044,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510144.590, "dur": 0.470,
    "args": {
      "External id": 2487045,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510146.915, "dur": 0.416,
    "args": {
      "External id": 2487046,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510148.827, "dur": 0.606,
    "args": {
      "External id": 2487047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510164.767, "dur": 34.766,
    "args": {
      "External id": 2487048,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510233.011, "dur": 96.360,
    "args": {
      "External id": 2487049,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510243.196, "dur": 2.962,
    "args": {
      "External id": 2487050,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510251.476, "dur": 12.192,
    "args": {
      "External id": 2487051,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510255.888, "dur": 7.321,
    "args": {
      "External id": 2487052,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510260.017, "dur": 2.052,
    "args": {
      "External id": 2487053,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510270.923, "dur": 23.420,
    "args": {
      "External id": 2487054,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510273.115, "dur": 0.426,
    "args": {
      "External id": 2487055,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510275.168, "dur": 0.556,
    "args": {
      "External id": 2487056,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510277.290, "dur": 0.349,
    "args": {
      "External id": 2487057,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510279.211, "dur": 0.545,
    "args": {
      "External id": 2487058,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510281.397, "dur": 0.378,
    "args": {
      "External id": 2487059,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510283.416, "dur": 0.387,
    "args": {
      "External id": 2487060,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510285.059, "dur": 0.386,
    "args": {
      "External id": 2487061,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510286.851, "dur": 1.812,
    "args": {
      "External id": 2487062,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510290.337, "dur": 0.370,
    "args": {
      "External id": 2487063,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510303.906, "dur": 18.179,
    "args": {
      "External id": 2487064,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510374.929, "dur": 107.758,
    "args": {
      "External id": 2487065,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510395.714, "dur": 83.767,
    "args": {
      "External id": 2487066,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3865, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510405.609, "dur": 69.790,
    "args": {
      "External id": 2487067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510495.724, "dur": 1.667,
    "args": {
      "External id": 2487068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3867, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510592.726, "dur": 1662.759,
    "args": {
      "External id": 2487069,"Sequence number": 24740446, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3868
    }
  },
  {
    "ph": "f", "id": 202, "pid": 1336760, "tid": 1381209, "ts": 1514657510592.726,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510705.260, "dur": 104.898,
    "args": {
      "External id": 2487070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510848.275, "dur": 41.208,
    "args": {
      "External id": 2487071,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510909.261, "dur": 48.496,
    "args": {
      "External id": 2487072,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657510967.168, "dur": 74.063,
    "args": {
      "External id": 2487073,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511052.725, "dur": 50.477,
    "args": {
      "External id": 2487074,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511115.041, "dur": 29.153,
    "args": {
      "External id": 2487075,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511153.835, "dur": 44.994,
    "args": {
      "External id": 2487076,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511223.495, "dur": 24.399,
    "args": {
      "External id": 2487077,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511266.503, "dur": 27.429,
    "args": {
      "External id": 2487078,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511312.324, "dur": 18.620,
    "args": {
      "External id": 2487079,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511345.759, "dur": 14.067,
    "args": {
      "External id": 2487080,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511370.965, "dur": 29.909,
    "args": {
      "External id": 2487081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511404.559, "dur": 32.834,
    "args": {
      "External id": 2487082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511462.313, "dur": 190.840,
    "args": {
      "External id": 2487083,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511556.934, "dur": 6.705,
    "args": {
      "External id": 2487084,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511565.789, "dur": 2.247,
    "args": {
      "External id": 2487085,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511691.072, "dur": 24.623,
    "args": {
      "External id": 2487086,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511727.472, "dur": 16.938,
    "args": {
      "External id": 2487087,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511755.189, "dur": 39.789,
    "args": {
      "External id": 2487088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511802.118, "dur": 34.520,
    "args": {
      "External id": 2487089,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511842.942, "dur": 20.582,
    "args": {
      "External id": 2487090,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511869.162, "dur": 29.860,
    "args": {
      "External id": 2487091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511907.605, "dur": 21.097,
    "args": {
      "External id": 2487092,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657511950.251, "dur": 77.257,
    "args": {
      "External id": 2487093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512050.755, "dur": 29.445,
    "args": {
      "External id": 2487094,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512099.272, "dur": 27.701,
    "args": {
      "External id": 2487095,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512143.162, "dur": 17.257,
    "args": {
      "External id": 2487096,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512175.957, "dur": 18.511,
    "args": {
      "External id": 2487097,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512206.599, "dur": 15.017,
    "args": {
      "External id": 2487098,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512303.443, "dur": 15.172,
    "args": {
      "External id": 2487099,"Record function id": 0, "Ev Idx": 3898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512306.509, "dur": 11.323,
    "args": {
      "External id": 2487100,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512310.935, "dur": 5.774,
    "args": {
      "External id": 2487101,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512312.505, "dur": 4.057,
    "args": {
      "External id": 2487102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512322.530, "dur": 5.386,
    "args": {
      "External id": 2487103,"Record function id": 0, "Ev Idx": 3902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512323.941, "dur": 3.487,
    "args": {
      "External id": 2487104,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512324.803, "dur": 2.132,
    "args": {
      "External id": 2487105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512325.817, "dur": 1.028,
    "args": {
      "External id": 2487106,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512331.246, "dur": 4.813,
    "args": {
      "External id": 2487107,"Record function id": 0, "Ev Idx": 3906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512332.694, "dur": 2.908,
    "args": {
      "External id": 2487108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512333.656, "dur": 1.367,
    "args": {
      "External id": 2487109,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512334.063, "dur": 0.894,
    "args": {
      "External id": 2487110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512339.221, "dur": 4.395,
    "args": {
      "External id": 2487111,"Record function id": 0, "Ev Idx": 3910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512340.614, "dur": 2.602,
    "args": {
      "External id": 2487112,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512341.457, "dur": 1.333,
    "args": {
      "External id": 2487113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512341.863, "dur": 0.863,
    "args": {
      "External id": 2487114,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512346.719, "dur": 3.788,
    "args": {
      "External id": 2487115,"Record function id": 0, "Ev Idx": 3914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512348.038, "dur": 2.055,
    "args": {
      "External id": 2487116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512348.532, "dur": 1.134,
    "args": {
      "External id": 2487117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512349.033, "dur": 0.568,
    "args": {
      "External id": 2487118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512353.739, "dur": 7.062,
    "args": {
      "External id": 2487119,"Record function id": 0, "Ev Idx": 3918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512354.705, "dur": 5.673,
    "args": {
      "External id": 2487120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512355.190, "dur": 4.456,
    "args": {
      "External id": 2487121,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512358.914, "dur": 0.633,
    "args": {
      "External id": 2487122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512364.106, "dur": 5.477,
    "args": {
      "External id": 2487123,"Record function id": 0, "Ev Idx": 3922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512365.205, "dur": 3.949,
    "args": {
      "External id": 2487124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512365.661, "dur": 2.933,
    "args": {
      "External id": 2487125,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512365.941, "dur": 2.426,
    "args": {
      "External id": 2487126,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512372.739, "dur": 3.784,
    "args": {
      "External id": 2487127,"Record function id": 0, "Ev Idx": 3926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512374.039, "dur": 2.076,
    "args": {
      "External id": 2487128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512374.555, "dur": 1.160,
    "args": {
      "External id": 2487129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512374.845, "dur": 0.803,
    "args": {
      "External id": 2487130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512379.592, "dur": 28.213,
    "args": {
      "External id": 2487131,"Record function id": 0, "Ev Idx": 3930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512380.508, "dur": 26.849,
    "args": {
      "External id": 2487132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512381.003, "dur": 25.754,
    "args": {
      "External id": 2487133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512405.973, "dur": 0.646,
    "args": {
      "External id": 2487134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512412.058, "dur": 35336.036,
    "args": {
      "External id": 2487135,"Record function id": 0, "Sequence number": 24740445, "Fwd thread id": 1, "Ev Idx": 3934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512413.637, "dur": 35324.618,
    "args": {
      "External id": 2487136,"Sequence number": 24740445, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3935
    }
  },
  {
    "ph": "f", "id": 203, "pid": 1336760, "tid": 1381209, "ts": 1514657512413.637,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512447.495, "dur": 39.631,
    "args": {
      "External id": 2487137,"Record function id": 0, "Ev Idx": 3936
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512495.168, "dur": 85.334,
    "args": {
      "External id": 2487138,"Record function id": 0, "Ev Idx": 3937
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.12)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512587.914, "dur": 35142.638,
    "args": {
      "External id": 2487139,"Record function id": 0, "Ev Idx": 3938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512677.690, "dur": 6.685,
    "args": {
      "External id": 2487140,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512694.892, "dur": 5.291,
    "args": {
      "External id": 2487141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512715.610, "dur": 34179.136,
    "args": {
      "External id": 2487142,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512728.824, "dur": 34156.747,
    "args": {
      "External id": 2487143,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512775.124, "dur": 12.939,
    "args": {
      "External id": 2487144,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512794.316, "dur": 34048.486,
    "args": {
      "External id": 2487145,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512796.915, "dur": 34045.181,
    "args": {
      "External id": 2487146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512801.118, "dur": 4.873,
    "args": {
      "External id": 2487147,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657512807.826, "dur": 34030.789,
    "args": {
      "External id": 2487148,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547010.596, "dur": 10.232,
    "args": {
      "External id": 2487149,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547014.137, "dur": 6.150,
    "args": {
      "External id": 2487150,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547053.188, "dur": 350.313,
    "args": {
      "External id": 2487151,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547079.834, "dur": 318.974,
    "args": {
      "External id": 2487152,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3951, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547092.029, "dur": 301.187,
    "args": {
      "External id": 2487153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547421.610, "dur": 2.225,
    "args": {
      "External id": 2487154,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3953, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547482.719, "dur": 7.831,
    "args": {
      "External id": 2487155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547561.959, "dur": 2.253,
    "args": {
      "External id": 2487156,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547584.296, "dur": 0.956,
    "args": {
      "External id": 2487157,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547598.719, "dur": 4.068,
    "args": {
      "External id": 2487158,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547614.622, "dur": 2.315,
    "args": {
      "External id": 2487159,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547629.123, "dur": 0.886,
    "args": {
      "External id": 2487160,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547641.597, "dur": 0.791,
    "args": {
      "External id": 2487161,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547653.894, "dur": 1.657,
    "args": {
      "External id": 2487162,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547665.681, "dur": 2.327,
    "args": {
      "External id": 2487163,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547764.159, "dur": 2741.222,
    "args": {
      "External id": 2487164,"Record function id": 0, "Ev Idx": 3963
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547783.063, "dur": 1033.042,
    "args": {
      "External id": 2487165,"Record function id": 0, "Ev Idx": 3964
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547799.234, "dur": 363.639,
    "args": {
      "External id": 2487166,"Record function id": 0, "Ev Idx": 3965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547884.435, "dur": 4.100,
    "args": {
      "External id": 2487167,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547891.746, "dur": 1.138,
    "args": {
      "External id": 2487168,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547894.391, "dur": 1.342,
    "args": {
      "External id": 2487169,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547897.259, "dur": 1.056,
    "args": {
      "External id": 2487170,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547899.575, "dur": 0.809,
    "args": {
      "External id": 2487171,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547902.183, "dur": 1.004,
    "args": {
      "External id": 2487172,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547904.629, "dur": 3.509,
    "args": {
      "External id": 2487173,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547909.322, "dur": 1.110,
    "args": {
      "External id": 2487174,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547911.701, "dur": 0.845,
    "args": {
      "External id": 2487175,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547914.105, "dur": 1.137,
    "args": {
      "External id": 2487176,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547932.141, "dur": 193.396,
    "args": {
      "External id": 2487177,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547946.856, "dur": 172.982,
    "args": {
      "External id": 2487178,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657547966.662, "dur": 12.046,
    "args": {
      "External id": 2487179,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548017.286, "dur": 71.923,
    "args": {
      "External id": 2487180,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548020.024, "dur": 68.815,
    "args": {
      "External id": 2487181,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548024.357, "dur": 6.752,
    "args": {
      "External id": 2487182,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548033.200, "dur": 55.127,
    "args": {
      "External id": 2487183,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3982
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.10", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548255.889, "dur": 552.127,
    "args": {
      "External id": 2487184,"Record function id": 0, "Ev Idx": 3983
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548273.680, "dur": 522.106,
    "args": {
      "External id": 2487185,"Record function id": 0, "Ev Idx": 3984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548337.504, "dur": 4.843,
    "args": {
      "External id": 2487186,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548358.889, "dur": 30.699,
    "args": {
      "External id": 2487187,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548363.946, "dur": 1.691,
    "args": {
      "External id": 2487188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548367.967, "dur": 2.359,
    "args": {
      "External id": 2487189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548372.128, "dur": 0.449,
    "args": {
      "External id": 2487190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548374.014, "dur": 0.434,
    "args": {
      "External id": 2487191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548376.206, "dur": 0.412,
    "args": {
      "External id": 2487192,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548378.048, "dur": 0.418,
    "args": {
      "External id": 2487193,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548380.330, "dur": 0.604,
    "args": {
      "External id": 2487194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548382.834, "dur": 0.358,
    "args": {
      "External id": 2487195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548385.001, "dur": 0.617,
    "args": {
      "External id": 2487196,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548399.815, "dur": 29.631,
    "args": {
      "External id": 2487197,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548459.329, "dur": 119.241,
    "args": {
      "External id": 2487198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548469.495, "dur": 4.726,
    "args": {
      "External id": 2487199,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548479.672, "dur": 10.454,
    "args": {
      "External id": 2487200,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548484.453, "dur": 5.265,
    "args": {
      "External id": 2487201,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548487.942, "dur": 0.666,
    "args": {
      "External id": 2487202,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548497.389, "dur": 24.044,
    "args": {
      "External id": 2487203,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548499.517, "dur": 0.694,
    "args": {
      "External id": 2487204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548501.820, "dur": 0.583,
    "args": {
      "External id": 2487205,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548503.996, "dur": 0.353,
    "args": {
      "External id": 2487206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548505.772, "dur": 0.341,
    "args": {
      "External id": 2487207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548507.614, "dur": 0.394,
    "args": {
      "External id": 2487208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548509.346, "dur": 0.419,
    "args": {
      "External id": 2487209,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548511.791, "dur": 2.242,
    "args": {
      "External id": 2487210,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548515.480, "dur": 0.353,
    "args": {
      "External id": 2487211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548517.473, "dur": 0.393,
    "args": {
      "External id": 2487212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548548.067, "dur": 22.130,
    "args": {
      "External id": 2487213,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548624.310, "dur": 110.234,
    "args": {
      "External id": 2487214,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548647.090, "dur": 84.147,
    "args": {
      "External id": 2487215,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4014, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548657.047, "dur": 70.049,
    "args": {
      "External id": 2487216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548746.870, "dur": 1.885,
    "args": {
      "External id": 2487217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4016, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548823.944, "dur": 1661.631,
    "args": {
      "External id": 2487218,"Sequence number": 24740444, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4017
    }
  },
  {
    "ph": "f", "id": 204, "pid": 1336760, "tid": 1381209, "ts": 1514657548823.944,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657548932.665, "dur": 149.858,
    "args": {
      "External id": 2487219,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549126.929, "dur": 41.283,
    "args": {
      "External id": 2487220,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549192.137, "dur": 56.969,
    "args": {
      "External id": 2487221,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549258.858, "dur": 33.717,
    "args": {
      "External id": 2487222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549299.605, "dur": 45.410,
    "args": {
      "External id": 2487223,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549354.820, "dur": 28.573,
    "args": {
      "External id": 2487224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549390.790, "dur": 43.144,
    "args": {
      "External id": 2487225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549455.500, "dur": 23.528,
    "args": {
      "External id": 2487226,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549497.653, "dur": 26.673,
    "args": {
      "External id": 2487227,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549564.958, "dur": 21.995,
    "args": {
      "External id": 2487228,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549600.960, "dur": 14.992,
    "args": {
      "External id": 2487229,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549627.236, "dur": 33.889,
    "args": {
      "External id": 2487230,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549664.714, "dur": 33.346,
    "args": {
      "External id": 2487231,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549724.407, "dur": 171.030,
    "args": {
      "External id": 2487232,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549802.016, "dur": 5.787,
    "args": {
      "External id": 2487233,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549809.996, "dur": 3.746,
    "args": {
      "External id": 2487234,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549929.069, "dur": 25.302,
    "args": {
      "External id": 2487235,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657549964.862, "dur": 13.853,
    "args": {
      "External id": 2487236,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550025.405, "dur": 44.517,
    "args": {
      "External id": 2487237,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550077.609, "dur": 35.988,
    "args": {
      "External id": 2487238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550120.979, "dur": 20.713,
    "args": {
      "External id": 2487239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550149.915, "dur": 30.244,
    "args": {
      "External id": 2487240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550187.259, "dur": 34.946,
    "args": {
      "External id": 2487241,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550235.560, "dur": 35.672,
    "args": {
      "External id": 2487242,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550291.119, "dur": 29.217,
    "args": {
      "External id": 2487243,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550337.616, "dur": 23.712,
    "args": {
      "External id": 2487244,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550378.845, "dur": 16.683,
    "args": {
      "External id": 2487245,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550410.011, "dur": 14.597,
    "args": {
      "External id": 2487246,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550438.771, "dur": 14.435,
    "args": {
      "External id": 2487247,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550549.446, "dur": 18.296,
    "args": {
      "External id": 2487248,"Record function id": 0, "Ev Idx": 4047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550553.785, "dur": 12.654,
    "args": {
      "External id": 2487249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550558.071, "dur": 6.927,
    "args": {
      "External id": 2487250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550560.266, "dur": 4.448,
    "args": {
      "External id": 2487251,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550572.912, "dur": 5.778,
    "args": {
      "External id": 2487252,"Record function id": 0, "Ev Idx": 4051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550574.479, "dur": 3.780,
    "args": {
      "External id": 2487253,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550575.408, "dur": 2.269,
    "args": {
      "External id": 2487254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550576.554, "dur": 1.028,
    "args": {
      "External id": 2487255,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550585.392, "dur": 5.099,
    "args": {
      "External id": 2487256,"Record function id": 0, "Ev Idx": 4055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550586.943, "dur": 3.150,
    "args": {
      "External id": 2487257,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550587.911, "dur": 1.725,
    "args": {
      "External id": 2487258,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550588.318, "dur": 1.252,
    "args": {
      "External id": 2487259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550593.766, "dur": 5.819,
    "args": {
      "External id": 2487260,"Record function id": 0, "Ev Idx": 4059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550595.107, "dur": 4.069,
    "args": {
      "External id": 2487261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550595.572, "dur": 3.169,
    "args": {
      "External id": 2487262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550596.017, "dur": 2.657,
    "args": {
      "External id": 2487263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550602.944, "dur": 3.926,
    "args": {
      "External id": 2487264,"Record function id": 0, "Ev Idx": 4063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550604.123, "dur": 2.361,
    "args": {
      "External id": 2487265,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550604.783, "dur": 1.243,
    "args": {
      "External id": 2487266,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550605.203, "dur": 0.761,
    "args": {
      "External id": 2487267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550610.357, "dur": 4.042,
    "args": {
      "External id": 2487268,"Record function id": 0, "Ev Idx": 4067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550611.337, "dur": 2.645,
    "args": {
      "External id": 2487269,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550612.110, "dur": 1.334,
    "args": {
      "External id": 2487270,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550612.598, "dur": 0.752,
    "args": {
      "External id": 2487271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550617.787, "dur": 4.087,
    "args": {
      "External id": 2487272,"Record function id": 0, "Ev Idx": 4071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550619.034, "dur": 2.440,
    "args": {
      "External id": 2487273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550619.819, "dur": 1.185,
    "args": {
      "External id": 2487274,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550620.254, "dur": 0.670,
    "args": {
      "External id": 2487275,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550625.120, "dur": 3.553,
    "args": {
      "External id": 2487276,"Record function id": 0, "Ev Idx": 4075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550626.091, "dur": 2.198,
    "args": {
      "External id": 2487277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550626.628, "dur": 1.254,
    "args": {
      "External id": 2487278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550626.979, "dur": 0.843,
    "args": {
      "External id": 2487279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550631.812, "dur": 4.202,
    "args": {
      "External id": 2487280,"Record function id": 0, "Ev Idx": 4079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550633.021, "dur": 2.612,
    "args": {
      "External id": 2487281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550633.917, "dur": 1.323,
    "args": {
      "External id": 2487282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550634.465, "dur": 0.683,
    "args": {
      "External id": 2487283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550639.912, "dur": 35585.474,
    "args": {
      "External id": 2487284,"Record function id": 0, "Sequence number": 24740443, "Fwd thread id": 1, "Ev Idx": 4083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550645.332, "dur": 35570.951,
    "args": {
      "External id": 2487285,"Sequence number": 24740443, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4084
    }
  },
  {
    "ph": "f", "id": 205, "pid": 1336760, "tid": 1381209, "ts": 1514657550645.332,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550674.748, "dur": 40.043,
    "args": {
      "External id": 2487286,"Record function id": 0, "Ev Idx": 4085
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550722.398, "dur": 68.608,
    "args": {
      "External id": 2487287,"Record function id": 0, "Ev Idx": 4086
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.11)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550796.819, "dur": 35411.614,
    "args": {
      "External id": 2487288,"Record function id": 0, "Ev Idx": 4087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550883.910, "dur": 6.425,
    "args": {
      "External id": 2487289,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550899.916, "dur": 4.857,
    "args": {
      "External id": 2487290,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550920.161, "dur": 34461.203,
    "args": {
      "External id": 2487291,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657550934.101, "dur": 34436.848,
    "args": {
      "External id": 2487292,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657551047.811, "dur": 16.126,
    "args": {
      "External id": 2487293,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657551070.597, "dur": 34262.195,
    "args": {
      "External id": 2487294,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657551073.209, "dur": 34258.429,
    "args": {
      "External id": 2487295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657551077.175, "dur": 6.618,
    "args": {
      "External id": 2487296,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657551110.655, "dur": 34217.730,
    "args": {
      "External id": 2487297,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585474.766, "dur": 9.058,
    "args": {
      "External id": 2487298,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585477.929, "dur": 5.584,
    "args": {
      "External id": 2487299,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585511.171, "dur": 369.413,
    "args": {
      "External id": 2487300,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585548.431, "dur": 326.870,
    "args": {
      "External id": 2487301,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4100, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585560.969, "dur": 308.977,
    "args": {
      "External id": 2487302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585900.265, "dur": 1.782,
    "args": {
      "External id": 2487303,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4102, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657585959.976, "dur": 6.514,
    "args": {
      "External id": 2487304,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586043.877, "dur": 1.993,
    "args": {
      "External id": 2487305,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586064.189, "dur": 0.991,
    "args": {
      "External id": 2487306,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586078.167, "dur": 0.914,
    "args": {
      "External id": 2487307,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586091.282, "dur": 0.968,
    "args": {
      "External id": 2487308,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586102.971, "dur": 0.947,
    "args": {
      "External id": 2487309,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586119.984, "dur": 1.007,
    "args": {
      "External id": 2487310,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586134.592, "dur": 1.778,
    "args": {
      "External id": 2487311,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586146.925, "dur": 0.944,
    "args": {
      "External id": 2487312,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586240.804, "dur": 2735.293,
    "args": {
      "External id": 2487313,"Record function id": 0, "Ev Idx": 4112
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586260.481, "dur": 1046.496,
    "args": {
      "External id": 2487314,"Record function id": 0, "Ev Idx": 4113
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586274.394, "dur": 330.710,
    "args": {
      "External id": 2487315,"Record function id": 0, "Ev Idx": 4114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586355.197, "dur": 4.183,
    "args": {
      "External id": 2487316,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586362.490, "dur": 1.413,
    "args": {
      "External id": 2487317,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586365.625, "dur": 1.110,
    "args": {
      "External id": 2487318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586368.260, "dur": 2.654,
    "args": {
      "External id": 2487319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586372.156, "dur": 1.211,
    "args": {
      "External id": 2487320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586375.132, "dur": 0.784,
    "args": {
      "External id": 2487321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586377.357, "dur": 1.685,
    "args": {
      "External id": 2487322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586380.163, "dur": 1.140,
    "args": {
      "External id": 2487323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586382.898, "dur": 1.385,
    "args": {
      "External id": 2487324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586385.592, "dur": 0.812,
    "args": {
      "External id": 2487325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586403.249, "dur": 168.570,
    "args": {
      "External id": 2487326,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586418.367, "dur": 148.253,
    "args": {
      "External id": 2487327,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586436.383, "dur": 11.933,
    "args": {
      "External id": 2487328,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586451.513, "dur": 68.975,
    "args": {
      "External id": 2487329,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586454.051, "dur": 66.021,
    "args": {
      "External id": 2487330,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586458.131, "dur": 7.195,
    "args": {
      "External id": 2487331,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586467.276, "dur": 52.175,
    "args": {
      "External id": 2487332,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4131
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.9", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586697.209, "dur": 602.339,
    "args": {
      "External id": 2487333,"Record function id": 0, "Ev Idx": 4132
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586714.585, "dur": 571.754,
    "args": {
      "External id": 2487334,"Record function id": 0, "Ev Idx": 4133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586781.641, "dur": 5.387,
    "args": {
      "External id": 2487335,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586803.177, "dur": 33.970,
    "args": {
      "External id": 2487336,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586808.587, "dur": 1.654,
    "args": {
      "External id": 2487337,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586812.407, "dur": 0.525,
    "args": {
      "External id": 2487338,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586814.672, "dur": 0.407,
    "args": {
      "External id": 2487339,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586816.749, "dur": 0.486,
    "args": {
      "External id": 2487340,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586819.047, "dur": 0.499,
    "args": {
      "External id": 2487341,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586821.112, "dur": 0.440,
    "args": {
      "External id": 2487342,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586827.327, "dur": 2.053,
    "args": {
      "External id": 2487343,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586831.021, "dur": 0.330,
    "args": {
      "External id": 2487344,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586832.759, "dur": 0.499,
    "args": {
      "External id": 2487345,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586847.248, "dur": 30.935,
    "args": {
      "External id": 2487346,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586909.489, "dur": 146.412,
    "args": {
      "External id": 2487347,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586919.489, "dur": 3.282,
    "args": {
      "External id": 2487348,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586928.094, "dur": 10.107,
    "args": {
      "External id": 2487349,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586932.603, "dur": 5.155,
    "args": {
      "External id": 2487350,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586935.970, "dur": 0.539,
    "args": {
      "External id": 2487351,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586947.675, "dur": 23.542,
    "args": {
      "External id": 2487352,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586950.036, "dur": 0.601,
    "args": {
      "External id": 2487353,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586952.270, "dur": 0.402,
    "args": {
      "External id": 2487354,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586954.195, "dur": 1.007,
    "args": {
      "External id": 2487355,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586956.624, "dur": 1.836,
    "args": {
      "External id": 2487356,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586960.115, "dur": 0.413,
    "args": {
      "External id": 2487357,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586962.023, "dur": 0.370,
    "args": {
      "External id": 2487358,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586964.124, "dur": 0.523,
    "args": {
      "External id": 2487359,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586966.173, "dur": 0.401,
    "args": {
      "External id": 2487360,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657586968.257, "dur": 0.277,
    "args": {
      "External id": 2487361,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587020.106, "dur": 26.107,
    "args": {
      "External id": 2487362,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587107.200, "dur": 111.098,
    "args": {
      "External id": 2487363,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587129.200, "dur": 85.811,
    "args": {
      "External id": 2487364,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4163, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587139.007, "dur": 71.460,
    "args": {
      "External id": 2487365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587233.913, "dur": 1.794,
    "args": {
      "External id": 2487366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4165, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587314.311, "dur": 1638.594,
    "args": {
      "External id": 2487367,"Sequence number": 24740442, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4166
    }
  },
  {
    "ph": "f", "id": 206, "pid": 1336760, "tid": 1381209, "ts": 1514657587314.311,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587426.281, "dur": 120.531,
    "args": {
      "External id": 2487368,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587588.308, "dur": 41.710,
    "args": {
      "External id": 2487369,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587647.037, "dur": 53.725,
    "args": {
      "External id": 2487370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587709.818, "dur": 33.858,
    "args": {
      "External id": 2487371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587752.451, "dur": 44.331,
    "args": {
      "External id": 2487372,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587803.898, "dur": 28.106,
    "args": {
      "External id": 2487373,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587839.243, "dur": 42.304,
    "args": {
      "External id": 2487374,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587902.050, "dur": 21.608,
    "args": {
      "External id": 2487375,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657587954.031, "dur": 75.232,
    "args": {
      "External id": 2487376,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588056.046, "dur": 22.710,
    "args": {
      "External id": 2487377,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588092.737, "dur": 14.748,
    "args": {
      "External id": 2487378,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588116.887, "dur": 35.995,
    "args": {
      "External id": 2487379,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588156.705, "dur": 33.883,
    "args": {
      "External id": 2487380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588220.266, "dur": 174.401,
    "args": {
      "External id": 2487381,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588300.688, "dur": 5.665,
    "args": {
      "External id": 2487382,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588308.700, "dur": 1.939,
    "args": {
      "External id": 2487383,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588427.602, "dur": 23.992,
    "args": {
      "External id": 2487384,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588463.110, "dur": 13.581,
    "args": {
      "External id": 2487385,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588485.400, "dur": 33.947,
    "args": {
      "External id": 2487386,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588526.150, "dur": 54.614,
    "args": {
      "External id": 2487387,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588592.140, "dur": 23.025,
    "args": {
      "External id": 2487388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588620.811, "dur": 30.220,
    "args": {
      "External id": 2487389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588657.643, "dur": 21.357,
    "args": {
      "External id": 2487390,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588687.649, "dur": 49.002,
    "args": {
      "External id": 2487391,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588760.168, "dur": 25.427,
    "args": {
      "External id": 2487392,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588802.902, "dur": 24.654,
    "args": {
      "External id": 2487393,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588846.208, "dur": 16.817,
    "args": {
      "External id": 2487394,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588878.789, "dur": 15.565,
    "args": {
      "External id": 2487395,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657588910.136, "dur": 14.842,
    "args": {
      "External id": 2487396,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589048.011, "dur": 17.489,
    "args": {
      "External id": 2487397,"Record function id": 0, "Ev Idx": 4196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589052.270, "dur": 11.985,
    "args": {
      "External id": 2487398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589056.497, "dur": 6.278,
    "args": {
      "External id": 2487399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589058.242, "dur": 4.315,
    "args": {
      "External id": 2487400,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589069.824, "dur": 5.239,
    "args": {
      "External id": 2487401,"Record function id": 0, "Ev Idx": 4200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589071.188, "dur": 3.441,
    "args": {
      "External id": 2487402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589072.148, "dur": 2.001,
    "args": {
      "External id": 2487403,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589072.994, "dur": 1.067,
    "args": {
      "External id": 2487404,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589078.487, "dur": 6.145,
    "args": {
      "External id": 2487405,"Record function id": 0, "Ev Idx": 4204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589079.692, "dur": 4.529,
    "args": {
      "External id": 2487406,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589080.630, "dur": 3.169,
    "args": {
      "External id": 2487407,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589081.140, "dur": 2.579,
    "args": {
      "External id": 2487408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589088.239, "dur": 4.452,
    "args": {
      "External id": 2487409,"Record function id": 0, "Ev Idx": 4208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589089.661, "dur": 2.522,
    "args": {
      "External id": 2487410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589090.336, "dur": 1.429,
    "args": {
      "External id": 2487411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589090.807, "dur": 0.882,
    "args": {
      "External id": 2487412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589095.950, "dur": 3.771,
    "args": {
      "External id": 2487413,"Record function id": 0, "Ev Idx": 4212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589097.051, "dur": 2.272,
    "args": {
      "External id": 2487414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589097.724, "dur": 1.119,
    "args": {
      "External id": 2487415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589098.022, "dur": 0.759,
    "args": {
      "External id": 2487416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589102.914, "dur": 4.193,
    "args": {
      "External id": 2487417,"Record function id": 0, "Ev Idx": 4216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589103.896, "dur": 2.782,
    "args": {
      "External id": 2487418,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589104.791, "dur": 1.445,
    "args": {
      "External id": 2487419,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589105.476, "dur": 0.645,
    "args": {
      "External id": 2487420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589110.473, "dur": 3.668,
    "args": {
      "External id": 2487421,"Record function id": 0, "Ev Idx": 4220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589111.570, "dur": 2.157,
    "args": {
      "External id": 2487422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589112.116, "dur": 1.183,
    "args": {
      "External id": 2487423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589112.591, "dur": 0.627,
    "args": {
      "External id": 2487424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589117.232, "dur": 3.758,
    "args": {
      "External id": 2487425,"Record function id": 0, "Ev Idx": 4224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589118.503, "dur": 2.052,
    "args": {
      "External id": 2487426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589118.994, "dur": 1.093,
    "args": {
      "External id": 2487427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589119.434, "dur": 0.587,
    "args": {
      "External id": 2487428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589124.027, "dur": 3.580,
    "args": {
      "External id": 2487429,"Record function id": 0, "Ev Idx": 4228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589124.963, "dur": 2.236,
    "args": {
      "External id": 2487430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589125.448, "dur": 1.330,
    "args": {
      "External id": 2487431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589126.140, "dur": 0.548,
    "args": {
      "External id": 2487432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589131.699, "dur": 35795.292,
    "args": {
      "External id": 2487433,"Record function id": 0, "Sequence number": 24740441, "Fwd thread id": 1, "Ev Idx": 4232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589133.013, "dur": 35784.612,
    "args": {
      "External id": 2487434,"Sequence number": 24740441, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4233
    }
  },
  {
    "ph": "f", "id": 207, "pid": 1336760, "tid": 1381209, "ts": 1514657589133.013,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589163.600, "dur": 39.296,
    "args": {
      "External id": 2487435,"Record function id": 0, "Ev Idx": 4234
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589210.138, "dur": 69.848,
    "args": {
      "External id": 2487436,"Record function id": 0, "Ev Idx": 4235
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.10)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589287.168, "dur": 35623.175,
    "args": {
      "External id": 2487437,"Record function id": 0, "Ev Idx": 4236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589373.888, "dur": 6.534,
    "args": {
      "External id": 2487438,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589390.027, "dur": 6.414,
    "args": {
      "External id": 2487439,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589409.453, "dur": 34644.388,
    "args": {
      "External id": 2487440,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589422.808, "dur": 34620.587,
    "args": {
      "External id": 2487441,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589481.701, "dur": 13.429,
    "args": {
      "External id": 2487442,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589501.637, "dur": 34466.255,
    "args": {
      "External id": 2487443,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589504.528, "dur": 34462.417,
    "args": {
      "External id": 2487444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589508.285, "dur": 5.149,
    "args": {
      "External id": 2487445,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657589515.220, "dur": 34447.327,
    "args": {
      "External id": 2487446,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624153.007, "dur": 10.065,
    "args": {
      "External id": 2487447,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624156.356, "dur": 6.379,
    "args": {
      "External id": 2487448,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624193.202, "dur": 423.588,
    "args": {
      "External id": 2487449,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624219.631, "dur": 391.533,
    "args": {
      "External id": 2487450,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4249, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624231.622, "dur": 373.338,
    "args": {
      "External id": 2487451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624638.622, "dur": 2.198,
    "args": {
      "External id": 2487452,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4251, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624703.586, "dur": 6.616,
    "args": {
      "External id": 2487453,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624754.853, "dur": 1.280,
    "args": {
      "External id": 2487454,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624771.819, "dur": 3.211,
    "args": {
      "External id": 2487455,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624787.244, "dur": 0.904,
    "args": {
      "External id": 2487456,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624800.108, "dur": 1.015,
    "args": {
      "External id": 2487457,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624812.150, "dur": 0.935,
    "args": {
      "External id": 2487458,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624824.128, "dur": 2.862,
    "args": {
      "External id": 2487459,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624838.015, "dur": 1.731,
    "args": {
      "External id": 2487460,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624849.937, "dur": 0.816,
    "args": {
      "External id": 2487461,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624940.927, "dur": 2742.076,
    "args": {
      "External id": 2487462,"Record function id": 0, "Ev Idx": 4261
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624961.494, "dur": 1074.322,
    "args": {
      "External id": 2487463,"Record function id": 0, "Ev Idx": 4262
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657624977.169, "dur": 364.611,
    "args": {
      "External id": 2487464,"Record function id": 0, "Ev Idx": 4263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625102.906, "dur": 4.888,
    "args": {
      "External id": 2487465,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625111.058, "dur": 1.004,
    "args": {
      "External id": 2487466,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625114.209, "dur": 2.826,
    "args": {
      "External id": 2487467,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625119.275, "dur": 0.875,
    "args": {
      "External id": 2487468,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625121.793, "dur": 1.113,
    "args": {
      "External id": 2487469,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625124.435, "dur": 1.161,
    "args": {
      "External id": 2487470,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625127.321, "dur": 1.304,
    "args": {
      "External id": 2487471,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625130.055, "dur": 1.239,
    "args": {
      "External id": 2487472,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625133.266, "dur": 0.788,
    "args": {
      "External id": 2487473,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625135.696, "dur": 0.899,
    "args": {
      "External id": 2487474,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625154.507, "dur": 157.353,
    "args": {
      "External id": 2487475,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625170.126, "dur": 137.094,
    "args": {
      "External id": 2487476,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625192.806, "dur": 14.491,
    "args": {
      "External id": 2487477,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625210.058, "dur": 70.453,
    "args": {
      "External id": 2487478,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625212.689, "dur": 67.513,
    "args": {
      "External id": 2487479,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625216.283, "dur": 5.509,
    "args": {
      "External id": 2487480,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625227.308, "dur": 52.199,
    "args": {
      "External id": 2487481,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4280
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.8", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625432.958, "dur": 593.892,
    "args": {
      "External id": 2487482,"Record function id": 0, "Ev Idx": 4281
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625450.545, "dur": 525.547,
    "args": {
      "External id": 2487483,"Record function id": 0, "Ev Idx": 4282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625515.629, "dur": 7.768,
    "args": {
      "External id": 2487484,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625557.493, "dur": 30.980,
    "args": {
      "External id": 2487485,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625562.630, "dur": 1.830,
    "args": {
      "External id": 2487486,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625566.723, "dur": 0.337,
    "args": {
      "External id": 2487487,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625568.457, "dur": 0.800,
    "args": {
      "External id": 2487488,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625570.726, "dur": 0.400,
    "args": {
      "External id": 2487489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625572.713, "dur": 0.483,
    "args": {
      "External id": 2487490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625574.517, "dur": 2.070,
    "args": {
      "External id": 2487491,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625578.079, "dur": 3.542,
    "args": {
      "External id": 2487492,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625582.706, "dur": 0.333,
    "args": {
      "External id": 2487493,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625584.283, "dur": 0.416,
    "args": {
      "External id": 2487494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625598.398, "dur": 32.779,
    "args": {
      "External id": 2487495,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625663.894, "dur": 98.850,
    "args": {
      "External id": 2487496,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625674.135, "dur": 3.578,
    "args": {
      "External id": 2487497,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625683.266, "dur": 9.779,
    "args": {
      "External id": 2487498,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625687.358, "dur": 5.271,
    "args": {
      "External id": 2487499,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625690.847, "dur": 0.656,
    "args": {
      "External id": 2487500,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625700.012, "dur": 26.488,
    "args": {
      "External id": 2487501,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625701.980, "dur": 0.588,
    "args": {
      "External id": 2487502,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625704.345, "dur": 0.481,
    "args": {
      "External id": 2487503,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625706.621, "dur": 2.155,
    "args": {
      "External id": 2487504,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625710.372, "dur": 0.500,
    "args": {
      "External id": 2487505,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625712.286, "dur": 0.410,
    "args": {
      "External id": 2487506,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625714.230, "dur": 0.670,
    "args": {
      "External id": 2487507,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625716.905, "dur": 0.601,
    "args": {
      "External id": 2487508,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625719.118, "dur": 0.411,
    "args": {
      "External id": 2487509,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625721.423, "dur": 0.298,
    "args": {
      "External id": 2487510,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625737.378, "dur": 18.132,
    "args": {
      "External id": 2487511,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625805.271, "dur": 106.221,
    "args": {
      "External id": 2487512,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625827.880, "dur": 80.275,
    "args": {
      "External id": 2487513,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4312, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625836.760, "dur": 67.263,
    "args": {
      "External id": 2487514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657625928.182, "dur": 2.094,
    "args": {
      "External id": 2487515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4314, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626044.436, "dur": 1618.335,
    "args": {
      "External id": 2487516,"Sequence number": 24740440, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4315
    }
  },
  {
    "ph": "f", "id": 208, "pid": 1336760, "tid": 1381209, "ts": 1514657626044.436,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626159.797, "dur": 106.511,
    "args": {
      "External id": 2487517,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626308.192, "dur": 39.486,
    "args": {
      "External id": 2487518,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626365.036, "dur": 49.197,
    "args": {
      "External id": 2487519,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626425.004, "dur": 32.425,
    "args": {
      "External id": 2487520,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626463.553, "dur": 44.887,
    "args": {
      "External id": 2487521,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626515.298, "dur": 45.182,
    "args": {
      "External id": 2487522,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626571.727, "dur": 44.655,
    "args": {
      "External id": 2487523,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626643.454, "dur": 22.793,
    "args": {
      "External id": 2487524,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626684.687, "dur": 24.798,
    "args": {
      "External id": 2487525,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626731.778, "dur": 17.197,
    "args": {
      "External id": 2487526,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626760.961, "dur": 14.080,
    "args": {
      "External id": 2487527,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626782.753, "dur": 28.482,
    "args": {
      "External id": 2487528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626814.555, "dur": 31.754,
    "args": {
      "External id": 2487529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626875.086, "dur": 205.976,
    "args": {
      "External id": 2487530,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626951.919, "dur": 6.599,
    "args": {
      "External id": 2487531,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657626960.740, "dur": 2.064,
    "args": {
      "External id": 2487532,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627117.262, "dur": 29.762,
    "args": {
      "External id": 2487533,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627158.043, "dur": 14.529,
    "args": {
      "External id": 2487534,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627181.721, "dur": 44.438,
    "args": {
      "External id": 2487535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627233.662, "dur": 38.224,
    "args": {
      "External id": 2487536,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627280.883, "dur": 20.765,
    "args": {
      "External id": 2487537,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627316.016, "dur": 30.173,
    "args": {
      "External id": 2487538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627352.833, "dur": 20.345,
    "args": {
      "External id": 2487539,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627380.480, "dur": 29.275,
    "args": {
      "External id": 2487540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627427.201, "dur": 34.230,
    "args": {
      "External id": 2487541,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627484.508, "dur": 27.185,
    "args": {
      "External id": 2487542,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627527.342, "dur": 35.124,
    "args": {
      "External id": 2487543,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627587.113, "dur": 16.214,
    "args": {
      "External id": 2487544,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627616.113, "dur": 15.829,
    "args": {
      "External id": 2487545,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627709.012, "dur": 15.192,
    "args": {
      "External id": 2487546,"Record function id": 0, "Ev Idx": 4345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627712.142, "dur": 11.006,
    "args": {
      "External id": 2487547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627716.046, "dur": 6.247,
    "args": {
      "External id": 2487548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627717.901, "dur": 4.217,
    "args": {
      "External id": 2487549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627728.345, "dur": 5.248,
    "args": {
      "External id": 2487550,"Record function id": 0, "Ev Idx": 4349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627729.694, "dur": 3.473,
    "args": {
      "External id": 2487551,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627730.539, "dur": 2.155,
    "args": {
      "External id": 2487552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627731.603, "dur": 1.004,
    "args": {
      "External id": 2487553,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627736.919, "dur": 4.487,
    "args": {
      "External id": 2487554,"Record function id": 0, "Ev Idx": 4353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627738.146, "dur": 2.844,
    "args": {
      "External id": 2487555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627738.953, "dur": 1.596,
    "args": {
      "External id": 2487556,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627739.471, "dur": 1.016,
    "args": {
      "External id": 2487557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627744.633, "dur": 5.660,
    "args": {
      "External id": 2487558,"Record function id": 0, "Ev Idx": 4357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627745.833, "dur": 4.050,
    "args": {
      "External id": 2487559,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627746.501, "dur": 2.963,
    "args": {
      "External id": 2487560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627746.867, "dur": 2.499,
    "args": {
      "External id": 2487561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627753.595, "dur": 4.468,
    "args": {
      "External id": 2487562,"Record function id": 0, "Ev Idx": 4361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627755.386, "dur": 2.273,
    "args": {
      "External id": 2487563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627756.033, "dur": 1.244,
    "args": {
      "External id": 2487564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627756.519, "dur": 0.693,
    "args": {
      "External id": 2487565,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627764.133, "dur": 4.197,
    "args": {
      "External id": 2487566,"Record function id": 0, "Ev Idx": 4365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627765.220, "dur": 2.680,
    "args": {
      "External id": 2487567,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627766.039, "dur": 1.427,
    "args": {
      "External id": 2487568,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627766.705, "dur": 0.673,
    "args": {
      "External id": 2487569,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627771.835, "dur": 3.945,
    "args": {
      "External id": 2487570,"Record function id": 0, "Ev Idx": 4369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627773.031, "dur": 2.340,
    "args": {
      "External id": 2487571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627773.626, "dur": 1.145,
    "args": {
      "External id": 2487572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627773.952, "dur": 0.751,
    "args": {
      "External id": 2487573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627778.956, "dur": 3.352,
    "args": {
      "External id": 2487574,"Record function id": 0, "Ev Idx": 4373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627779.866, "dur": 2.027,
    "args": {
      "External id": 2487575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627780.339, "dur": 1.174,
    "args": {
      "External id": 2487576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627780.627, "dur": 0.781,
    "args": {
      "External id": 2487577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627785.416, "dur": 4.214,
    "args": {
      "External id": 2487578,"Record function id": 0, "Ev Idx": 4377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627786.438, "dur": 2.763,
    "args": {
      "External id": 2487579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627787.011, "dur": 1.514,
    "args": {
      "External id": 2487580,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627787.696, "dur": 0.715,
    "args": {
      "External id": 2487581,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627793.457, "dur": 35037.047,
    "args": {
      "External id": 2487582,"Record function id": 0, "Sequence number": 24740439, "Fwd thread id": 1, "Ev Idx": 4381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627794.814, "dur": 35027.232,
    "args": {
      "External id": 2487583,"Sequence number": 24740439, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4382
    }
  },
  {
    "ph": "f", "id": 209, "pid": 1336760, "tid": 1381209, "ts": 1514657627794.814,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627825.578, "dur": 38.023,
    "args": {
      "External id": 2487584,"Record function id": 0, "Ev Idx": 4383
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627871.986, "dur": 67.995,
    "args": {
      "External id": 2487585,"Record function id": 0, "Ev Idx": 4384
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.9)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657627949.061, "dur": 34864.715,
    "args": {
      "External id": 2487586,"Record function id": 0, "Ev Idx": 4385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628086.096, "dur": 7.255,
    "args": {
      "External id": 2487587,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628103.948, "dur": 5.171,
    "args": {
      "External id": 2487588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628124.112, "dur": 33882.125,
    "args": {
      "External id": 2487589,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628141.161, "dur": 33832.405,
    "args": {
      "External id": 2487590,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628191.311, "dur": 15.398,
    "args": {
      "External id": 2487591,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628212.796, "dur": 33719.818,
    "args": {
      "External id": 2487592,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628215.317, "dur": 33716.608,
    "args": {
      "External id": 2487593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628219.063, "dur": 4.847,
    "args": {
      "External id": 2487594,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657628225.663, "dur": 33702.741,
    "args": {
      "External id": 2487595,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662099.688, "dur": 10.891,
    "args": {
      "External id": 2487596,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662104.433, "dur": 5.788,
    "args": {
      "External id": 2487597,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662139.285, "dur": 370.214,
    "args": {
      "External id": 2487598,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662167.635, "dur": 336.983,
    "args": {
      "External id": 2487599,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4398, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662179.306, "dur": 319.365,
    "args": {
      "External id": 2487600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662545.649, "dur": 3.359,
    "args": {
      "External id": 2487601,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4400, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662610.649, "dur": 6.678,
    "args": {
      "External id": 2487602,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662660.228, "dur": 1.383,
    "args": {
      "External id": 2487603,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662677.078, "dur": 1.320,
    "args": {
      "External id": 2487604,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662691.357, "dur": 1.120,
    "args": {
      "External id": 2487605,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662704.831, "dur": 0.694,
    "args": {
      "External id": 2487606,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662715.619, "dur": 0.729,
    "args": {
      "External id": 2487607,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662727.309, "dur": 0.964,
    "args": {
      "External id": 2487608,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662740.880, "dur": 1.726,
    "args": {
      "External id": 2487609,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662752.966, "dur": 0.763,
    "args": {
      "External id": 2487610,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662845.858, "dur": 2766.139,
    "args": {
      "External id": 2487611,"Record function id": 0, "Ev Idx": 4410
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662865.463, "dur": 1042.122,
    "args": {
      "External id": 2487612,"Record function id": 0, "Ev Idx": 4411
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662879.361, "dur": 361.822,
    "args": {
      "External id": 2487613,"Record function id": 0, "Ev Idx": 4412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662965.213, "dur": 4.101,
    "args": {
      "External id": 2487614,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662972.374, "dur": 1.062,
    "args": {
      "External id": 2487615,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662977.182, "dur": 1.133,
    "args": {
      "External id": 2487616,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657662980.415, "dur": 35.955,
    "args": {
      "External id": 2487617,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663021.307, "dur": 2.821,
    "args": {
      "External id": 2487618,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663025.726, "dur": 1.369,
    "args": {
      "External id": 2487619,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663030.433, "dur": 1.056,
    "args": {
      "External id": 2487620,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663033.195, "dur": 0.890,
    "args": {
      "External id": 2487621,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663035.570, "dur": 1.241,
    "args": {
      "External id": 2487622,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663038.330, "dur": 0.835,
    "args": {
      "External id": 2487623,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663059.419, "dur": 152.558,
    "args": {
      "External id": 2487624,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663076.313, "dur": 131.077,
    "args": {
      "External id": 2487625,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663095.033, "dur": 12.941,
    "args": {
      "External id": 2487626,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663111.052, "dur": 68.482,
    "args": {
      "External id": 2487627,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663113.785, "dur": 65.430,
    "args": {
      "External id": 2487628,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663117.899, "dur": 6.708,
    "args": {
      "External id": 2487629,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663126.204, "dur": 52.165,
    "args": {
      "External id": 2487630,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4429
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.7", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663325.699, "dur": 573.966,
    "args": {
      "External id": 2487631,"Record function id": 0, "Ev Idx": 4430
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663343.236, "dur": 544.385,
    "args": {
      "External id": 2487632,"Record function id": 0, "Ev Idx": 4431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663405.298, "dur": 6.326,
    "args": {
      "External id": 2487633,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663427.469, "dur": 35.697,
    "args": {
      "External id": 2487634,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663436.199, "dur": 3.005,
    "args": {
      "External id": 2487635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663441.377, "dur": 0.873,
    "args": {
      "External id": 2487636,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663443.942, "dur": 0.643,
    "args": {
      "External id": 2487637,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663446.715, "dur": 0.455,
    "args": {
      "External id": 2487638,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663448.631, "dur": 0.288,
    "args": {
      "External id": 2487639,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663450.314, "dur": 0.557,
    "args": {
      "External id": 2487640,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663452.648, "dur": 0.313,
    "args": {
      "External id": 2487641,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663454.181, "dur": 2.076,
    "args": {
      "External id": 2487642,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663458.290, "dur": 0.665,
    "args": {
      "External id": 2487643,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663472.440, "dur": 29.717,
    "args": {
      "External id": 2487644,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663551.289, "dur": 113.264,
    "args": {
      "External id": 2487645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663562.383, "dur": 4.603,
    "args": {
      "External id": 2487646,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663572.818, "dur": 10.592,
    "args": {
      "External id": 2487647,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663576.864, "dur": 6.143,
    "args": {
      "External id": 2487648,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663580.455, "dur": 0.876,
    "args": {
      "External id": 2487649,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663590.991, "dur": 28.078,
    "args": {
      "External id": 2487650,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663593.371, "dur": 0.703,
    "args": {
      "External id": 2487651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663595.628, "dur": 0.578,
    "args": {
      "External id": 2487652,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663597.709, "dur": 0.685,
    "args": {
      "External id": 2487653,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663599.784, "dur": 0.983,
    "args": {
      "External id": 2487654,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663602.230, "dur": 1.960,
    "args": {
      "External id": 2487655,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663605.707, "dur": 0.300,
    "args": {
      "External id": 2487656,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663607.487, "dur": 0.228,
    "args": {
      "External id": 2487657,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663609.084, "dur": 0.596,
    "args": {
      "External id": 2487658,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663611.548, "dur": 0.472,
    "args": {
      "External id": 2487659,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663634.583, "dur": 21.868,
    "args": {
      "External id": 2487660,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663708.935, "dur": 116.221,
    "args": {
      "External id": 2487661,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663736.925, "dur": 84.908,
    "args": {
      "External id": 2487662,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4461, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663746.739, "dur": 70.505,
    "args": {
      "External id": 2487663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663841.564, "dur": 1.854,
    "args": {
      "External id": 2487664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4463, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657663915.474, "dur": 1677.047,
    "args": {
      "External id": 2487665,"Sequence number": 24740438, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4464
    }
  },
  {
    "ph": "f", "id": 210, "pid": 1336760, "tid": 1381209, "ts": 1514657663915.474,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664068.636, "dur": 105.267,
    "args": {
      "External id": 2487666,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664218.054, "dur": 40.775,
    "args": {
      "External id": 2487667,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664274.713, "dur": 49.025,
    "args": {
      "External id": 2487668,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664333.038, "dur": 32.732,
    "args": {
      "External id": 2487669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664371.784, "dur": 45.150,
    "args": {
      "External id": 2487670,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664425.262, "dur": 27.561,
    "args": {
      "External id": 2487671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664460.079, "dur": 40.828,
    "args": {
      "External id": 2487672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664524.922, "dur": 41.975,
    "args": {
      "External id": 2487673,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664588.869, "dur": 26.687,
    "args": {
      "External id": 2487674,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664636.823, "dur": 18.677,
    "args": {
      "External id": 2487675,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664672.533, "dur": 15.004,
    "args": {
      "External id": 2487676,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664695.192, "dur": 31.731,
    "args": {
      "External id": 2487677,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664729.987, "dur": 33.263,
    "args": {
      "External id": 2487678,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664790.214, "dur": 160.407,
    "args": {
      "External id": 2487679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664863.143, "dur": 7.011,
    "args": {
      "External id": 2487680,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657664872.187, "dur": 2.369,
    "args": {
      "External id": 2487681,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665035.259, "dur": 27.673,
    "args": {
      "External id": 2487682,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665076.744, "dur": 13.922,
    "args": {
      "External id": 2487683,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665100.286, "dur": 53.980,
    "args": {
      "External id": 2487684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665160.091, "dur": 36.218,
    "args": {
      "External id": 2487685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665203.469, "dur": 19.939,
    "args": {
      "External id": 2487686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665231.006, "dur": 31.679,
    "args": {
      "External id": 2487687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665268.470, "dur": 19.386,
    "args": {
      "External id": 2487688,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665298.711, "dur": 42.810,
    "args": {
      "External id": 2487689,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665370.347, "dur": 22.538,
    "args": {
      "External id": 2487690,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665414.072, "dur": 28.088,
    "args": {
      "External id": 2487691,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665457.943, "dur": 16.041,
    "args": {
      "External id": 2487692,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665490.737, "dur": 16.557,
    "args": {
      "External id": 2487693,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665522.807, "dur": 36.713,
    "args": {
      "External id": 2487694,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665637.616, "dur": 15.222,
    "args": {
      "External id": 2487695,"Record function id": 0, "Ev Idx": 4494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665640.620, "dur": 11.295,
    "args": {
      "External id": 2487696,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665644.791, "dur": 6.221,
    "args": {
      "External id": 2487697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665646.592, "dur": 4.224,
    "args": {
      "External id": 2487698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665657.155, "dur": 5.625,
    "args": {
      "External id": 2487699,"Record function id": 0, "Ev Idx": 4498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665658.535, "dur": 3.824,
    "args": {
      "External id": 2487700,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665659.916, "dur": 2.008,
    "args": {
      "External id": 2487701,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665660.682, "dur": 1.099,
    "args": {
      "External id": 2487702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665666.415, "dur": 4.716,
    "args": {
      "External id": 2487703,"Record function id": 0, "Ev Idx": 4502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665667.491, "dur": 3.214,
    "args": {
      "External id": 2487704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665668.310, "dur": 1.796,
    "args": {
      "External id": 2487705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665668.918, "dur": 1.084,
    "args": {
      "External id": 2487706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665677.666, "dur": 4.047,
    "args": {
      "External id": 2487707,"Record function id": 0, "Ev Idx": 4506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665678.962, "dur": 2.358,
    "args": {
      "External id": 2487708,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665679.747, "dur": 1.146,
    "args": {
      "External id": 2487709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665680.209, "dur": 0.622,
    "args": {
      "External id": 2487710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665684.825, "dur": 5.919,
    "args": {
      "External id": 2487711,"Record function id": 0, "Ev Idx": 4510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665685.955, "dur": 4.396,
    "args": {
      "External id": 2487712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665686.716, "dur": 3.235,
    "args": {
      "External id": 2487713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665686.986, "dur": 2.902,
    "args": {
      "External id": 2487714,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665694.031, "dur": 4.200,
    "args": {
      "External id": 2487715,"Record function id": 0, "Ev Idx": 4514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665695.260, "dur": 2.538,
    "args": {
      "External id": 2487716,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665695.765, "dur": 1.617,
    "args": {
      "External id": 2487717,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665696.372, "dur": 0.914,
    "args": {
      "External id": 2487718,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665701.582, "dur": 3.577,
    "args": {
      "External id": 2487719,"Record function id": 0, "Ev Idx": 4518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665702.658, "dur": 2.111,
    "args": {
      "External id": 2487720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665703.128, "dur": 1.108,
    "args": {
      "External id": 2487721,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665703.432, "dur": 0.712,
    "args": {
      "External id": 2487722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665708.302, "dur": 4.249,
    "args": {
      "External id": 2487723,"Record function id": 0, "Ev Idx": 4522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665709.524, "dur": 2.633,
    "args": {
      "External id": 2487724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665710.090, "dur": 1.496,
    "args": {
      "External id": 2487725,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665710.709, "dur": 0.785,
    "args": {
      "External id": 2487726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665715.617, "dur": 4.683,
    "args": {
      "External id": 2487727,"Record function id": 0, "Ev Idx": 4526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665716.672, "dur": 3.205,
    "args": {
      "External id": 2487728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665717.528, "dur": 1.811,
    "args": {
      "External id": 2487729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665718.335, "dur": 0.885,
    "args": {
      "External id": 2487730,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665723.758, "dur": 36320.017,
    "args": {
      "External id": 2487731,"Record function id": 0, "Sequence number": 24740437, "Fwd thread id": 1, "Ev Idx": 4530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665725.177, "dur": 36309.264,
    "args": {
      "External id": 2487732,"Sequence number": 24740437, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4531
    }
  },
  {
    "ph": "f", "id": 211, "pid": 1336760, "tid": 1381209, "ts": 1514657665725.177,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665757.791, "dur": 37.644,
    "args": {
      "External id": 2487733,"Record function id": 0, "Ev Idx": 4532
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665806.292, "dur": 65.049,
    "args": {
      "External id": 2487734,"Record function id": 0, "Ev Idx": 4533
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.8)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665877.226, "dur": 36148.683,
    "args": {
      "External id": 2487735,"Record function id": 0, "Ev Idx": 4534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657665966.875, "dur": 7.032,
    "args": {
      "External id": 2487736,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666024.344, "dur": 5.543,
    "args": {
      "External id": 2487737,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666050.204, "dur": 35085.379,
    "args": {
      "External id": 2487738,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666063.831, "dur": 35062.322,
    "args": {
      "External id": 2487739,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666115.359, "dur": 14.355,
    "args": {
      "External id": 2487740,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666136.453, "dur": 34953.473,
    "args": {
      "External id": 2487741,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666139.295, "dur": 34949.687,
    "args": {
      "External id": 2487742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666143.265, "dur": 7.180,
    "args": {
      "External id": 2487743,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657666152.063, "dur": 34932.824,
    "args": {
      "External id": 2487744,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701228.403, "dur": 8.832,
    "args": {
      "External id": 2487745,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701231.142, "dur": 5.742,
    "args": {
      "External id": 2487746,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701264.908, "dur": 434.279,
    "args": {
      "External id": 2487747,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701293.170, "dur": 400.583,
    "args": {
      "External id": 2487748,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4547, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701304.534, "dur": 383.245,
    "args": {
      "External id": 2487749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701723.112, "dur": 2.311,
    "args": {
      "External id": 2487750,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4549, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701785.859, "dur": 6.628,
    "args": {
      "External id": 2487751,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701836.422, "dur": 1.523,
    "args": {
      "External id": 2487752,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701854.420, "dur": 1.302,
    "args": {
      "External id": 2487753,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701867.644, "dur": 2.634,
    "args": {
      "External id": 2487754,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701884.945, "dur": 1.523,
    "args": {
      "External id": 2487755,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701897.813, "dur": 1.229,
    "args": {
      "External id": 2487756,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701909.577, "dur": 1.065,
    "args": {
      "External id": 2487757,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701922.272, "dur": 2.849,
    "args": {
      "External id": 2487758,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657701934.540, "dur": 1.000,
    "args": {
      "External id": 2487759,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702059.708, "dur": 2723.394,
    "args": {
      "External id": 2487760,"Record function id": 0, "Ev Idx": 4559
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702079.445, "dur": 1064.896,
    "args": {
      "External id": 2487761,"Record function id": 0, "Ev Idx": 4560
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702094.291, "dur": 313.691,
    "args": {
      "External id": 2487762,"Record function id": 0, "Ev Idx": 4561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702179.855, "dur": 5.001,
    "args": {
      "External id": 2487763,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702187.778, "dur": 1.423,
    "args": {
      "External id": 2487764,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702190.841, "dur": 1.340,
    "args": {
      "External id": 2487765,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702195.680, "dur": 0.973,
    "args": {
      "External id": 2487766,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702198.119, "dur": 1.158,
    "args": {
      "External id": 2487767,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702200.706, "dur": 2.528,
    "args": {
      "External id": 2487768,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702204.735, "dur": 0.830,
    "args": {
      "External id": 2487769,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702208.500, "dur": 0.924,
    "args": {
      "External id": 2487770,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702211.516, "dur": 0.917,
    "args": {
      "External id": 2487771,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702214.027, "dur": 1.143,
    "args": {
      "External id": 2487772,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702232.746, "dur": 148.259,
    "args": {
      "External id": 2487773,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702248.627, "dur": 128.218,
    "args": {
      "External id": 2487774,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702266.655, "dur": 12.677,
    "args": {
      "External id": 2487775,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702283.866, "dur": 66.553,
    "args": {
      "External id": 2487776,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702286.397, "dur": 63.669,
    "args": {
      "External id": 2487777,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702290.316, "dur": 6.130,
    "args": {
      "External id": 2487778,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702298.222, "dur": 51.197,
    "args": {
      "External id": 2487779,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4578
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.6", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702492.265, "dur": 643.957,
    "args": {
      "External id": 2487780,"Record function id": 0, "Ev Idx": 4579
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702508.633, "dur": 614.856,
    "args": {
      "External id": 2487781,"Record function id": 0, "Ev Idx": 4580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702587.872, "dur": 6.157,
    "args": {
      "External id": 2487782,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702615.004, "dur": 39.926,
    "args": {
      "External id": 2487783,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702619.649, "dur": 4.400,
    "args": {
      "External id": 2487784,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702626.320, "dur": 0.770,
    "args": {
      "External id": 2487785,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702628.399, "dur": 0.417,
    "args": {
      "External id": 2487786,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702631.141, "dur": 0.619,
    "args": {
      "External id": 2487787,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702633.180, "dur": 0.633,
    "args": {
      "External id": 2487788,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702635.109, "dur": 0.575,
    "args": {
      "External id": 2487789,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702637.342, "dur": 0.479,
    "args": {
      "External id": 2487790,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702639.345, "dur": 0.451,
    "args": {
      "External id": 2487791,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702647.184, "dur": 2.667,
    "args": {
      "External id": 2487792,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702664.718, "dur": 32.459,
    "args": {
      "External id": 2487793,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702727.522, "dur": 116.701,
    "args": {
      "External id": 2487794,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702737.685, "dur": 3.429,
    "args": {
      "External id": 2487795,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702746.340, "dur": 10.065,
    "args": {
      "External id": 2487796,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702750.526, "dur": 5.481,
    "args": {
      "External id": 2487797,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702754.178, "dur": 0.432,
    "args": {
      "External id": 2487798,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702763.826, "dur": 22.975,
    "args": {
      "External id": 2487799,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702765.835, "dur": 0.535,
    "args": {
      "External id": 2487800,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702768.300, "dur": 0.544,
    "args": {
      "External id": 2487801,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702770.134, "dur": 0.355,
    "args": {
      "External id": 2487802,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702771.883, "dur": 0.593,
    "args": {
      "External id": 2487803,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702774.023, "dur": 0.790,
    "args": {
      "External id": 2487804,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702776.266, "dur": 1.857,
    "args": {
      "External id": 2487805,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702779.373, "dur": 0.519,
    "args": {
      "External id": 2487806,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702781.006, "dur": 0.567,
    "args": {
      "External id": 2487807,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702783.125, "dur": 0.387,
    "args": {
      "External id": 2487808,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702815.187, "dur": 21.262,
    "args": {
      "External id": 2487809,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702888.118, "dur": 161.794,
    "args": {
      "External id": 2487810,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702918.133, "dur": 127.616,
    "args": {
      "External id": 2487811,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4610, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657702928.160, "dur": 112.739,
    "args": {
      "External id": 2487812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703067.305, "dur": 1.912,
    "args": {
      "External id": 2487813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4612, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703152.046, "dur": 1608.150,
    "args": {
      "External id": 2487814,"Sequence number": 24740436, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4613
    }
  },
  {
    "ph": "f", "id": 212, "pid": 1336760, "tid": 1381209, "ts": 1514657703152.046,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703265.584, "dur": 106.195,
    "args": {
      "External id": 2487815,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703412.943, "dur": 39.786,
    "args": {
      "External id": 2487816,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703468.719, "dur": 46.306,
    "args": {
      "External id": 2487817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703524.066, "dur": 54.482,
    "args": {
      "External id": 2487818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703592.878, "dur": 46.230,
    "args": {
      "External id": 2487819,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703647.880, "dur": 27.644,
    "args": {
      "External id": 2487820,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703682.983, "dur": 41.124,
    "args": {
      "External id": 2487821,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703749.597, "dur": 22.432,
    "args": {
      "External id": 2487822,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703792.021, "dur": 25.808,
    "args": {
      "External id": 2487823,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703838.498, "dur": 19.584,
    "args": {
      "External id": 2487824,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703872.311, "dur": 14.167,
    "args": {
      "External id": 2487825,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703895.285, "dur": 27.785,
    "args": {
      "External id": 2487826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657703925.973, "dur": 31.835,
    "args": {
      "External id": 2487827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704022.948, "dur": 181.803,
    "args": {
      "External id": 2487828,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704107.754, "dur": 8.689,
    "args": {
      "External id": 2487829,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704118.627, "dur": 2.083,
    "args": {
      "External id": 2487830,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704240.278, "dur": 25.648,
    "args": {
      "External id": 2487831,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704277.225, "dur": 13.780,
    "args": {
      "External id": 2487832,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704298.738, "dur": 40.311,
    "args": {
      "External id": 2487833,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704344.235, "dur": 33.838,
    "args": {
      "External id": 2487834,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704384.308, "dur": 19.938,
    "args": {
      "External id": 2487835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704409.890, "dur": 28.943,
    "args": {
      "External id": 2487836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704444.117, "dur": 19.774,
    "args": {
      "External id": 2487837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704470.936, "dur": 28.688,
    "args": {
      "External id": 2487838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704553.510, "dur": 30.939,
    "args": {
      "External id": 2487839,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704609.040, "dur": 22.588,
    "args": {
      "External id": 2487840,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704648.801, "dur": 19.872,
    "args": {
      "External id": 2487841,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704686.401, "dur": 13.902,
    "args": {
      "External id": 2487842,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704714.976, "dur": 18.329,
    "args": {
      "External id": 2487843,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704804.950, "dur": 15.397,
    "args": {
      "External id": 2487844,"Record function id": 0, "Ev Idx": 4643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704808.259, "dur": 11.154,
    "args": {
      "External id": 2487845,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704812.320, "dur": 6.240,
    "args": {
      "External id": 2487846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704814.014, "dur": 4.400,
    "args": {
      "External id": 2487847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704824.427, "dur": 5.038,
    "args": {
      "External id": 2487848,"Record function id": 0, "Ev Idx": 4647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704826.099, "dur": 2.915,
    "args": {
      "External id": 2487849,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704826.769, "dur": 1.763,
    "args": {
      "External id": 2487850,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704827.428, "dur": 0.999,
    "args": {
      "External id": 2487851,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704832.754, "dur": 7.224,
    "args": {
      "External id": 2487852,"Record function id": 0, "Ev Idx": 4651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704834.086, "dur": 5.426,
    "args": {
      "External id": 2487853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704834.676, "dur": 1.283,
    "args": {
      "External id": 2487854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704835.183, "dur": 0.710,
    "args": {
      "External id": 2487855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704843.258, "dur": 4.040,
    "args": {
      "External id": 2487856,"Record function id": 0, "Ev Idx": 4655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704844.407, "dur": 2.496,
    "args": {
      "External id": 2487857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704845.036, "dur": 1.329,
    "args": {
      "External id": 2487858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704845.648, "dur": 0.651,
    "args": {
      "External id": 2487859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704850.457, "dur": 5.567,
    "args": {
      "External id": 2487860,"Record function id": 0, "Ev Idx": 4659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704851.503, "dur": 4.065,
    "args": {
      "External id": 2487861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704852.018, "dur": 3.091,
    "args": {
      "External id": 2487862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704852.626, "dur": 2.376,
    "args": {
      "External id": 2487863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704859.217, "dur": 3.700,
    "args": {
      "External id": 2487864,"Record function id": 0, "Ev Idx": 4663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704860.429, "dur": 2.049,
    "args": {
      "External id": 2487865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704860.891, "dur": 1.191,
    "args": {
      "External id": 2487866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704861.204, "dur": 0.812,
    "args": {
      "External id": 2487867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704866.176, "dur": 3.444,
    "args": {
      "External id": 2487868,"Record function id": 0, "Ev Idx": 4667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704867.227, "dur": 1.987,
    "args": {
      "External id": 2487869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704867.697, "dur": 1.089,
    "args": {
      "External id": 2487870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704868.013, "dur": 0.694,
    "args": {
      "External id": 2487871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704872.728, "dur": 3.626,
    "args": {
      "External id": 2487872,"Record function id": 0, "Ev Idx": 4671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704873.790, "dur": 2.123,
    "args": {
      "External id": 2487873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704874.329, "dur": 1.166,
    "args": {
      "External id": 2487874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704874.767, "dur": 0.596,
    "args": {
      "External id": 2487875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704879.528, "dur": 4.633,
    "args": {
      "External id": 2487876,"Record function id": 0, "Ev Idx": 4675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704881.089, "dur": 2.670,
    "args": {
      "External id": 2487877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704881.549, "dur": 1.689,
    "args": {
      "External id": 2487878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704882.251, "dur": 0.927,
    "args": {
      "External id": 2487879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704887.636, "dur": 35484.466,
    "args": {
      "External id": 2487880,"Record function id": 0, "Sequence number": 24740435, "Fwd thread id": 1, "Ev Idx": 4679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704889.465, "dur": 35473.668,
    "args": {
      "External id": 2487881,"Sequence number": 24740435, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4680
    }
  },
  {
    "ph": "f", "id": 213, "pid": 1336760, "tid": 1381209, "ts": 1514657704889.465,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704921.189, "dur": 36.794,
    "args": {
      "External id": 2487882,"Record function id": 0, "Ev Idx": 4681
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657704964.679, "dur": 105.429,
    "args": {
      "External id": 2487883,"Record function id": 0, "Ev Idx": 4682
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.7)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705077.433, "dur": 35278.409,
    "args": {
      "External id": 2487884,"Record function id": 0, "Ev Idx": 4683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705168.254, "dur": 7.275,
    "args": {
      "External id": 2487885,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705185.726, "dur": 5.445,
    "args": {
      "External id": 2487886,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705204.460, "dur": 34420.976,
    "args": {
      "External id": 2487887,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705218.118, "dur": 34398.619,
    "args": {
      "External id": 2487888,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705268.004, "dur": 16.456,
    "args": {
      "External id": 2487889,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705290.687, "dur": 34284.679,
    "args": {
      "External id": 2487890,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705293.554, "dur": 34281.121,
    "args": {
      "External id": 2487891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705297.622, "dur": 7.245,
    "args": {
      "External id": 2487892,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657705306.542, "dur": 34264.790,
    "args": {
      "External id": 2487893,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657739713.760, "dur": 8.823,
    "args": {
      "External id": 2487894,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657739716.262, "dur": 5.986,
    "args": {
      "External id": 2487895,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657739752.112, "dur": 322.783,
    "args": {
      "External id": 2487896,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657739780.577, "dur": 289.256,
    "args": {
      "External id": 2487897,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4696, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657739792.137, "dur": 271.714,
    "args": {
      "External id": 2487898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740098.568, "dur": 1.944,
    "args": {
      "External id": 2487899,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4698, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740157.819, "dur": 6.913,
    "args": {
      "External id": 2487900,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740208.936, "dur": 1.470,
    "args": {
      "External id": 2487901,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740225.033, "dur": 1.455,
    "args": {
      "External id": 2487902,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740237.676, "dur": 3.042,
    "args": {
      "External id": 2487903,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740252.947, "dur": 0.743,
    "args": {
      "External id": 2487904,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740263.720, "dur": 0.956,
    "args": {
      "External id": 2487905,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740274.380, "dur": 0.960,
    "args": {
      "External id": 2487906,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740286.211, "dur": 2.668,
    "args": {
      "External id": 2487907,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740300.400, "dur": 1.007,
    "args": {
      "External id": 2487908,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740386.469, "dur": 2757.314,
    "args": {
      "External id": 2487909,"Record function id": 0, "Ev Idx": 4708
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740404.851, "dur": 1047.773,
    "args": {
      "External id": 2487910,"Record function id": 0, "Ev Idx": 4709
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740418.155, "dur": 331.901,
    "args": {
      "External id": 2487911,"Record function id": 0, "Ev Idx": 4710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740498.439, "dur": 3.976,
    "args": {
      "External id": 2487912,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740505.503, "dur": 1.276,
    "args": {
      "External id": 2487913,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740508.498, "dur": 0.910,
    "args": {
      "External id": 2487914,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740511.343, "dur": 0.796,
    "args": {
      "External id": 2487915,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740513.848, "dur": 0.989,
    "args": {
      "External id": 2487916,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740517.552, "dur": 2.902,
    "args": {
      "External id": 2487917,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740522.114, "dur": 0.983,
    "args": {
      "External id": 2487918,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740524.868, "dur": 0.903,
    "args": {
      "External id": 2487919,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740527.289, "dur": 16.519,
    "args": {
      "External id": 2487920,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740548.571, "dur": 1.175,
    "args": {
      "External id": 2487921,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740569.169, "dur": 154.282,
    "args": {
      "External id": 2487922,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740584.671, "dur": 134.369,
    "args": {
      "External id": 2487923,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740605.611, "dur": 12.001,
    "args": {
      "External id": 2487924,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740620.456, "dur": 69.896,
    "args": {
      "External id": 2487925,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740622.655, "dur": 67.316,
    "args": {
      "External id": 2487926,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740629.449, "dur": 6.989,
    "args": {
      "External id": 2487927,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740638.011, "dur": 51.207,
    "args": {
      "External id": 2487928,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740835.111, "dur": 609.840,
    "args": {
      "External id": 2487929,"Record function id": 0, "Ev Idx": 4728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740851.817, "dur": 580.978,
    "args": {
      "External id": 2487930,"Record function id": 0, "Ev Idx": 4729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740912.359, "dur": 4.637,
    "args": {
      "External id": 2487931,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740936.781, "dur": 30.009,
    "args": {
      "External id": 2487932,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740941.762, "dur": 3.267,
    "args": {
      "External id": 2487933,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740947.067, "dur": 0.713,
    "args": {
      "External id": 2487934,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740949.114, "dur": 0.824,
    "args": {
      "External id": 2487935,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740951.061, "dur": 0.589,
    "args": {
      "External id": 2487936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740953.182, "dur": 0.354,
    "args": {
      "External id": 2487937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740955.272, "dur": 0.389,
    "args": {
      "External id": 2487938,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740956.977, "dur": 0.517,
    "args": {
      "External id": 2487939,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740958.976, "dur": 0.442,
    "args": {
      "External id": 2487940,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740960.903, "dur": 2.625,
    "args": {
      "External id": 2487941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657740977.203, "dur": 84.223,
    "args": {
      "External id": 2487942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741096.764, "dur": 110.058,
    "args": {
      "External id": 2487943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741107.348, "dur": 4.130,
    "args": {
      "External id": 2487944,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741116.673, "dur": 11.051,
    "args": {
      "External id": 2487945,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741120.835, "dur": 6.476,
    "args": {
      "External id": 2487946,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741124.896, "dur": 0.808,
    "args": {
      "External id": 2487947,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741135.255, "dur": 27.184,
    "args": {
      "External id": 2487948,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741137.459, "dur": 0.589,
    "args": {
      "External id": 2487949,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741139.997, "dur": 0.550,
    "args": {
      "External id": 2487950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741144.292, "dur": 0.569,
    "args": {
      "External id": 2487951,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741146.268, "dur": 0.536,
    "args": {
      "External id": 2487952,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741148.224, "dur": 0.540,
    "args": {
      "External id": 2487953,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741150.125, "dur": 2.312,
    "args": {
      "External id": 2487954,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741154.026, "dur": 0.302,
    "args": {
      "External id": 2487955,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741156.137, "dur": 0.372,
    "args": {
      "External id": 2487956,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741157.795, "dur": 0.527,
    "args": {
      "External id": 2487957,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741178.575, "dur": 20.844,
    "args": {
      "External id": 2487958,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741253.075, "dur": 116.230,
    "args": {
      "External id": 2487959,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741281.874, "dur": 84.243,
    "args": {
      "External id": 2487960,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4759, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741291.955, "dur": 70.201,
    "args": {
      "External id": 2487961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741384.875, "dur": 1.781,
    "args": {
      "External id": 2487962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4761, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741459.936, "dur": 1664.525,
    "args": {
      "External id": 2487963,"Sequence number": 24740434, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4762
    }
  },
  {
    "ph": "f", "id": 214, "pid": 1336760, "tid": 1381209, "ts": 1514657741459.936,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741590.831, "dur": 104.440,
    "args": {
      "External id": 2487964,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741738.612, "dur": 42.185,
    "args": {
      "External id": 2487965,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741796.799, "dur": 48.603,
    "args": {
      "External id": 2487966,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741854.781, "dur": 31.393,
    "args": {
      "External id": 2487967,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741892.056, "dur": 44.114,
    "args": {
      "External id": 2487968,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741944.631, "dur": 27.591,
    "args": {
      "External id": 2487969,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657741980.384, "dur": 85.663,
    "args": {
      "External id": 2487970,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742095.704, "dur": 25.672,
    "args": {
      "External id": 2487971,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742141.366, "dur": 26.575,
    "args": {
      "External id": 2487972,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742188.833, "dur": 18.565,
    "args": {
      "External id": 2487973,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742222.843, "dur": 14.188,
    "args": {
      "External id": 2487974,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742245.480, "dur": 30.474,
    "args": {
      "External id": 2487975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742279.408, "dur": 33.648,
    "args": {
      "External id": 2487976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742344.317, "dur": 169.307,
    "args": {
      "External id": 2487977,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742417.937, "dur": 8.190,
    "args": {
      "External id": 2487978,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742428.016, "dur": 1.997,
    "args": {
      "External id": 2487979,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742571.984, "dur": 26.895,
    "args": {
      "External id": 2487980,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742610.184, "dur": 16.952,
    "args": {
      "External id": 2487981,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742635.460, "dur": 40.988,
    "args": {
      "External id": 2487982,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742681.918, "dur": 38.689,
    "args": {
      "External id": 2487983,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742726.902, "dur": 23.149,
    "args": {
      "External id": 2487984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742755.682, "dur": 29.517,
    "args": {
      "External id": 2487985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742790.633, "dur": 22.719,
    "args": {
      "External id": 2487986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742820.917, "dur": 29.490,
    "args": {
      "External id": 2487987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742871.655, "dur": 21.631,
    "args": {
      "External id": 2487988,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742927.602, "dur": 30.424,
    "args": {
      "External id": 2487989,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657742972.638, "dur": 49.625,
    "args": {
      "External id": 2487990,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743047.874, "dur": 16.430,
    "args": {
      "External id": 2487991,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743078.374, "dur": 18.350,
    "args": {
      "External id": 2487992,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743169.618, "dur": 15.455,
    "args": {
      "External id": 2487993,"Record function id": 0, "Ev Idx": 4792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743173.011, "dur": 11.160,
    "args": {
      "External id": 2487994,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743177.221, "dur": 6.005,
    "args": {
      "External id": 2487995,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743178.568, "dur": 4.545,
    "args": {
      "External id": 2487996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743189.049, "dur": 4.540,
    "args": {
      "External id": 2487997,"Record function id": 0, "Ev Idx": 4796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743190.503, "dur": 2.631,
    "args": {
      "External id": 2487998,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743191.302, "dur": 1.353,
    "args": {
      "External id": 2487999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743191.883, "dur": 0.704,
    "args": {
      "External id": 2488000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743196.814, "dur": 4.149,
    "args": {
      "External id": 2488001,"Record function id": 0, "Ev Idx": 4800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743197.990, "dur": 2.564,
    "args": {
      "External id": 2488002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743198.751, "dur": 1.370,
    "args": {
      "External id": 2488003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743199.122, "dur": 0.933,
    "args": {
      "External id": 2488004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743204.264, "dur": 3.994,
    "args": {
      "External id": 2488005,"Record function id": 0, "Ev Idx": 4804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743205.460, "dur": 2.327,
    "args": {
      "External id": 2488006,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743206.023, "dur": 1.320,
    "args": {
      "External id": 2488007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743206.567, "dur": 0.699,
    "args": {
      "External id": 2488008,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743214.705, "dur": 5.544,
    "args": {
      "External id": 2488009,"Record function id": 0, "Ev Idx": 4808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743215.770, "dur": 4.042,
    "args": {
      "External id": 2488010,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743216.353, "dur": 2.899,
    "args": {
      "External id": 2488011,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743216.761, "dur": 2.372,
    "args": {
      "External id": 2488012,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743223.451, "dur": 3.668,
    "args": {
      "External id": 2488013,"Record function id": 0, "Ev Idx": 4812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743224.415, "dur": 2.293,
    "args": {
      "External id": 2488014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743224.904, "dur": 1.201,
    "args": {
      "External id": 2488015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743225.199, "dur": 0.813,
    "args": {
      "External id": 2488016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743230.484, "dur": 4.855,
    "args": {
      "External id": 2488017,"Record function id": 0, "Ev Idx": 4816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743231.521, "dur": 3.395,
    "args": {
      "External id": 2488018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743231.957, "dur": 2.487,
    "args": {
      "External id": 2488019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743233.811, "dur": 0.533,
    "args": {
      "External id": 2488020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743238.504, "dur": 3.830,
    "args": {
      "External id": 2488021,"Record function id": 0, "Ev Idx": 4820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743239.680, "dur": 2.234,
    "args": {
      "External id": 2488022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743240.160, "dur": 1.217,
    "args": {
      "External id": 2488023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743240.628, "dur": 0.681,
    "args": {
      "External id": 2488024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743245.453, "dur": 3.667,
    "args": {
      "External id": 2488025,"Record function id": 0, "Ev Idx": 4824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743246.735, "dur": 1.966,
    "args": {
      "External id": 2488026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743247.161, "dur": 1.100,
    "args": {
      "External id": 2488027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743247.533, "dur": 0.634,
    "args": {
      "External id": 2488028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743252.933, "dur": 36097.135,
    "args": {
      "External id": 2488029,"Record function id": 0, "Sequence number": 24740433, "Fwd thread id": 1, "Ev Idx": 4828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743254.388, "dur": 36087.512,
    "args": {
      "External id": 2488030,"Sequence number": 24740433, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4829
    }
  },
  {
    "ph": "f", "id": 215, "pid": 1336760, "tid": 1381209, "ts": 1514657743254.388,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743283.869, "dur": 36.712,
    "args": {
      "External id": 2488031,"Record function id": 0, "Ev Idx": 4830
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743327.674, "dur": 63.325,
    "args": {
      "External id": 2488032,"Record function id": 0, "Ev Idx": 4831
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.6)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743396.868, "dur": 35937.462,
    "args": {
      "External id": 2488033,"Record function id": 0, "Ev Idx": 4832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743484.973, "dur": 6.402,
    "args": {
      "External id": 2488034,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743500.538, "dur": 5.002,
    "args": {
      "External id": 2488035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743522.754, "dur": 34979.046,
    "args": {
      "External id": 2488036,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743553.359, "dur": 34939.233,
    "args": {
      "External id": 2488037,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743605.910, "dur": 14.267,
    "args": {
      "External id": 2488038,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743626.690, "dur": 34828.284,
    "args": {
      "External id": 2488039,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743629.492, "dur": 34824.788,
    "args": {
      "External id": 2488040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743633.451, "dur": 6.161,
    "args": {
      "External id": 2488041,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657743641.403, "dur": 34809.602,
    "args": {
      "External id": 2488042,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657778603.464, "dur": 8.597,
    "args": {
      "External id": 2488043,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657778606.071, "dur": 5.539,
    "args": {
      "External id": 2488044,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657778640.778, "dur": 403.561,
    "args": {
      "External id": 2488045,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657778668.343, "dur": 370.385,
    "args": {
      "External id": 2488046,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4845, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657778679.898, "dur": 352.726,
    "args": {
      "External id": 2488047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779067.481, "dur": 2.648,
    "args": {
      "External id": 2488048,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4847, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779133.634, "dur": 8.409,
    "args": {
      "External id": 2488049,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779185.808, "dur": 1.143,
    "args": {
      "External id": 2488050,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779201.994, "dur": 1.350,
    "args": {
      "External id": 2488051,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779214.165, "dur": 1.530,
    "args": {
      "External id": 2488052,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779226.909, "dur": 2.476,
    "args": {
      "External id": 2488053,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779239.251, "dur": 0.839,
    "args": {
      "External id": 2488054,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779250.210, "dur": 0.973,
    "args": {
      "External id": 2488055,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779261.566, "dur": 0.789,
    "args": {
      "External id": 2488056,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779272.703, "dur": 2.303,
    "args": {
      "External id": 2488057,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779363.902, "dur": 2778.582,
    "args": {
      "External id": 2488058,"Record function id": 0, "Ev Idx": 4857
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779383.508, "dur": 1043.669,
    "args": {
      "External id": 2488059,"Record function id": 0, "Ev Idx": 4858
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779397.221, "dur": 330.427,
    "args": {
      "External id": 2488060,"Record function id": 0, "Ev Idx": 4859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779477.344, "dur": 4.096,
    "args": {
      "External id": 2488061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779484.201, "dur": 1.007,
    "args": {
      "External id": 2488062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779486.878, "dur": 0.963,
    "args": {
      "External id": 2488063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779489.663, "dur": 1.000,
    "args": {
      "External id": 2488064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779493.165, "dur": 0.838,
    "args": {
      "External id": 2488065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779495.443, "dur": 0.954,
    "args": {
      "External id": 2488066,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779497.980, "dur": 0.869,
    "args": {
      "External id": 2488067,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779500.302, "dur": 2.707,
    "args": {
      "External id": 2488068,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779505.449, "dur": 0.745,
    "args": {
      "External id": 2488069,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779507.568, "dur": 0.788,
    "args": {
      "External id": 2488070,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779525.048, "dur": 172.667,
    "args": {
      "External id": 2488071,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779560.286, "dur": 132.540,
    "args": {
      "External id": 2488072,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779580.490, "dur": 14.646,
    "args": {
      "External id": 2488073,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779598.253, "dur": 67.336,
    "args": {
      "External id": 2488074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779600.534, "dur": 64.724,
    "args": {
      "External id": 2488075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779605.271, "dur": 6.359,
    "args": {
      "External id": 2488076,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779614.377, "dur": 50.430,
    "args": {
      "External id": 2488077,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4876
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779812.653, "dur": 605.678,
    "args": {
      "External id": 2488078,"Record function id": 0, "Ev Idx": 4877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779835.097, "dur": 571.554,
    "args": {
      "External id": 2488079,"Record function id": 0, "Ev Idx": 4878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779895.223, "dur": 4.846,
    "args": {
      "External id": 2488080,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779919.421, "dur": 30.666,
    "args": {
      "External id": 2488081,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779925.026, "dur": 1.659,
    "args": {
      "External id": 2488082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779928.641, "dur": 0.830,
    "args": {
      "External id": 2488083,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779932.335, "dur": 2.359,
    "args": {
      "External id": 2488084,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779935.959, "dur": 0.415,
    "args": {
      "External id": 2488085,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779938.216, "dur": 0.585,
    "args": {
      "External id": 2488086,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779940.667, "dur": 0.362,
    "args": {
      "External id": 2488087,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779942.312, "dur": 0.736,
    "args": {
      "External id": 2488088,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779944.322, "dur": 0.846,
    "args": {
      "External id": 2488089,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779946.792, "dur": 0.522,
    "args": {
      "External id": 2488090,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657779960.109, "dur": 80.677,
    "args": {
      "External id": 2488091,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780076.837, "dur": 107.430,
    "args": {
      "External id": 2488092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780087.540, "dur": 4.410,
    "args": {
      "External id": 2488093,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780097.564, "dur": 11.908,
    "args": {
      "External id": 2488094,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780101.613, "dur": 7.466,
    "args": {
      "External id": 2488095,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780105.093, "dur": 2.425,
    "args": {
      "External id": 2488096,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780116.867, "dur": 27.870,
    "args": {
      "External id": 2488097,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780119.388, "dur": 0.682,
    "args": {
      "External id": 2488098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780121.999, "dur": 0.622,
    "args": {
      "External id": 2488099,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780123.987, "dur": 0.523,
    "args": {
      "External id": 2488100,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780125.855, "dur": 0.510,
    "args": {
      "External id": 2488101,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780128.159, "dur": 0.342,
    "args": {
      "External id": 2488102,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780130.451, "dur": 0.260,
    "args": {
      "External id": 2488103,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780132.022, "dur": 0.436,
    "args": {
      "External id": 2488104,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780133.811, "dur": 2.097,
    "args": {
      "External id": 2488105,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780137.498, "dur": 0.435,
    "args": {
      "External id": 2488106,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780155.685, "dur": 21.505,
    "args": {
      "External id": 2488107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780228.370, "dur": 113.290,
    "args": {
      "External id": 2488108,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780254.472, "dur": 83.946,
    "args": {
      "External id": 2488109,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4908, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780264.423, "dur": 69.830,
    "args": {
      "External id": 2488110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780357.581, "dur": 1.979,
    "args": {
      "External id": 2488111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4910, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780434.094, "dur": 1689.457,
    "args": {
      "External id": 2488112,"Sequence number": 24740432, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4911
    }
  },
  {
    "ph": "f", "id": 216, "pid": 1336760, "tid": 1381209, "ts": 1514657780434.094,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780561.517, "dur": 105.674,
    "args": {
      "External id": 2488113,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780709.057, "dur": 40.529,
    "args": {
      "External id": 2488114,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780767.103, "dur": 50.565,
    "args": {
      "External id": 2488115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780826.962, "dur": 34.023,
    "args": {
      "External id": 2488116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780867.015, "dur": 44.571,
    "args": {
      "External id": 2488117,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780917.833, "dur": 27.268,
    "args": {
      "External id": 2488118,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657780954.868, "dur": 77.130,
    "args": {
      "External id": 2488119,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781060.957, "dur": 23.061,
    "args": {
      "External id": 2488120,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781102.988, "dur": 30.174,
    "args": {
      "External id": 2488121,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781159.901, "dur": 17.976,
    "args": {
      "External id": 2488122,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781191.117, "dur": 17.357,
    "args": {
      "External id": 2488123,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781219.414, "dur": 33.381,
    "args": {
      "External id": 2488124,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781256.132, "dur": 33.255,
    "args": {
      "External id": 2488125,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781317.036, "dur": 161.683,
    "args": {
      "External id": 2488126,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781391.035, "dur": 5.731,
    "args": {
      "External id": 2488127,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781398.549, "dur": 3.273,
    "args": {
      "External id": 2488128,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781512.581, "dur": 45.064,
    "args": {
      "External id": 2488129,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781580.880, "dur": 15.028,
    "args": {
      "External id": 2488130,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781608.165, "dur": 39.477,
    "args": {
      "External id": 2488131,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781655.026, "dur": 35.063,
    "args": {
      "External id": 2488132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781696.775, "dur": 21.159,
    "args": {
      "External id": 2488133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781722.220, "dur": 41.520,
    "args": {
      "External id": 2488134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781776.678, "dur": 28.679,
    "args": {
      "External id": 2488135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781817.119, "dur": 35.506,
    "args": {
      "External id": 2488136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781879.539, "dur": 23.504,
    "args": {
      "External id": 2488137,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781924.003, "dur": 29.592,
    "args": {
      "External id": 2488138,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657781969.146, "dur": 52.964,
    "args": {
      "External id": 2488139,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782042.842, "dur": 20.334,
    "args": {
      "External id": 2488140,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782076.116, "dur": 16.006,
    "args": {
      "External id": 2488141,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782168.064, "dur": 14.919,
    "args": {
      "External id": 2488142,"Record function id": 0, "Ev Idx": 4941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782171.465, "dur": 10.539,
    "args": {
      "External id": 2488143,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782175.589, "dur": 5.443,
    "args": {
      "External id": 2488144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782177.001, "dur": 3.934,
    "args": {
      "External id": 2488145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782186.977, "dur": 6.395,
    "args": {
      "External id": 2488146,"Record function id": 0, "Ev Idx": 4945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782188.471, "dur": 4.457,
    "args": {
      "External id": 2488147,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782189.318, "dur": 3.132,
    "args": {
      "External id": 2488148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782189.863, "dur": 2.508,
    "args": {
      "External id": 2488149,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782196.663, "dur": 4.482,
    "args": {
      "External id": 2488150,"Record function id": 0, "Ev Idx": 4949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782197.924, "dur": 2.768,
    "args": {
      "External id": 2488151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782198.579, "dur": 1.593,
    "args": {
      "External id": 2488152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782199.188, "dur": 0.900,
    "args": {
      "External id": 2488153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782204.370, "dur": 3.593,
    "args": {
      "External id": 2488154,"Record function id": 0, "Ev Idx": 4953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782205.559, "dur": 1.988,
    "args": {
      "External id": 2488155,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782206.051, "dur": 1.085,
    "args": {
      "External id": 2488156,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782206.458, "dur": 0.570,
    "args": {
      "External id": 2488157,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782211.136, "dur": 3.560,
    "args": {
      "External id": 2488158,"Record function id": 0, "Ev Idx": 4957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782212.501, "dur": 1.798,
    "args": {
      "External id": 2488159,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782212.991, "dur": 0.899,
    "args": {
      "External id": 2488160,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782213.266, "dur": 0.555,
    "args": {
      "External id": 2488161,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782217.930, "dur": 3.956,
    "args": {
      "External id": 2488162,"Record function id": 0, "Ev Idx": 4961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782219.149, "dur": 2.341,
    "args": {
      "External id": 2488163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782219.668, "dur": 1.401,
    "args": {
      "External id": 2488164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782220.215, "dur": 0.787,
    "args": {
      "External id": 2488165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782225.216, "dur": 4.471,
    "args": {
      "External id": 2488166,"Record function id": 0, "Ev Idx": 4965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782226.718, "dur": 2.559,
    "args": {
      "External id": 2488167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782227.545, "dur": 1.284,
    "args": {
      "External id": 2488168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782228.114, "dur": 0.648,
    "args": {
      "External id": 2488169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782232.909, "dur": 3.899,
    "args": {
      "External id": 2488170,"Record function id": 0, "Ev Idx": 4969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782234.219, "dur": 2.196,
    "args": {
      "External id": 2488171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782234.829, "dur": 1.164,
    "args": {
      "External id": 2488172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782235.088, "dur": 0.844,
    "args": {
      "External id": 2488173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782239.888, "dur": 6.604,
    "args": {
      "External id": 2488174,"Record function id": 0, "Ev Idx": 4973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782241.071, "dur": 5.021,
    "args": {
      "External id": 2488175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782241.516, "dur": 4.146,
    "args": {
      "External id": 2488176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782242.894, "dur": 2.672,
    "args": {
      "External id": 2488177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782250.028, "dur": 38475.145,
    "args": {
      "External id": 2488178,"Record function id": 0, "Sequence number": 24740431, "Fwd thread id": 1, "Ev Idx": 4977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782251.552, "dur": 38464.813,
    "args": {
      "External id": 2488179,"Sequence number": 24740431, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4978
    }
  },
  {
    "ph": "f", "id": 217, "pid": 1336760, "tid": 1381209, "ts": 1514657782251.552,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782280.958, "dur": 36.185,
    "args": {
      "External id": 2488180,"Record function id": 0, "Ev Idx": 4979
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782324.827, "dur": 65.072,
    "args": {
      "External id": 2488181,"Record function id": 0, "Ev Idx": 4980
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.5)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782395.526, "dur": 38311.710,
    "args": {
      "External id": 2488182,"Record function id": 0, "Ev Idx": 4981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782488.455, "dur": 6.948,
    "args": {
      "External id": 2488183,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782505.008, "dur": 5.005,
    "args": {
      "External id": 2488184,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782523.498, "dur": 37280.806,
    "args": {
      "External id": 2488185,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782553.399, "dur": 37241.066,
    "args": {
      "External id": 2488186,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782602.403, "dur": 13.999,
    "args": {
      "External id": 2488187,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782622.618, "dur": 37134.720,
    "args": {
      "External id": 2488188,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782625.234, "dur": 37131.402,
    "args": {
      "External id": 2488189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782629.658, "dur": 5.700,
    "args": {
      "External id": 2488190,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657782636.918, "dur": 37115.667,
    "args": {
      "External id": 2488191,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657819897.209, "dur": 8.671,
    "args": {
      "External id": 2488192,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657819899.943, "dur": 5.612,
    "args": {
      "External id": 2488193,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657819934.293, "dur": 456.990,
    "args": {
      "External id": 2488194,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657819963.426, "dur": 422.463,
    "args": {
      "External id": 2488195,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4994, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657819974.262, "dur": 405.808,
    "args": {
      "External id": 2488196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820415.929, "dur": 2.457,
    "args": {
      "External id": 2488197,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4996, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820483.233, "dur": 6.626,
    "args": {
      "External id": 2488198,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820551.547, "dur": 2.292,
    "args": {
      "External id": 2488199,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820573.908, "dur": 1.192,
    "args": {
      "External id": 2488200,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820586.649, "dur": 1.122,
    "args": {
      "External id": 2488201,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820598.298, "dur": 1.053,
    "args": {
      "External id": 2488202,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820609.126, "dur": 1.060,
    "args": {
      "External id": 2488203,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820622.183, "dur": 1.279,
    "args": {
      "External id": 2488204,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820635.048, "dur": 1.091,
    "args": {
      "External id": 2488205,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820645.634, "dur": 0.937,
    "args": {
      "External id": 2488206,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820740.191, "dur": 2750.989,
    "args": {
      "External id": 2488207,"Record function id": 0, "Ev Idx": 5006
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820759.802, "dur": 1039.925,
    "args": {
      "External id": 2488208,"Record function id": 0, "Ev Idx": 5007
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820774.564, "dur": 363.938,
    "args": {
      "External id": 2488209,"Record function id": 0, "Ev Idx": 5008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820857.956, "dur": 4.184,
    "args": {
      "External id": 2488210,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820865.136, "dur": 1.239,
    "args": {
      "External id": 2488211,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820868.287, "dur": 2.833,
    "args": {
      "External id": 2488212,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820874.021, "dur": 0.874,
    "args": {
      "External id": 2488213,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820876.373, "dur": 0.892,
    "args": {
      "External id": 2488214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820878.552, "dur": 0.630,
    "args": {
      "External id": 2488215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820880.685, "dur": 0.848,
    "args": {
      "External id": 2488216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820884.358, "dur": 0.714,
    "args": {
      "External id": 2488217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820886.420, "dur": 1.584,
    "args": {
      "External id": 2488218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820889.516, "dur": 0.925,
    "args": {
      "External id": 2488219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820907.711, "dur": 193.991,
    "args": {
      "External id": 2488220,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820923.827, "dur": 172.654,
    "args": {
      "External id": 2488221,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820946.333, "dur": 15.019,
    "args": {
      "External id": 2488222,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820964.121, "dur": 102.749,
    "args": {
      "External id": 2488223,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820966.535, "dur": 99.991,
    "args": {
      "External id": 2488224,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820970.292, "dur": 6.028,
    "args": {
      "External id": 2488225,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657820977.868, "dur": 87.554,
    "args": {
      "External id": 2488226,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5025
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821231.490, "dur": 560.807,
    "args": {
      "External id": 2488227,"Record function id": 0, "Ev Idx": 5026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821248.882, "dur": 531.255,
    "args": {
      "External id": 2488228,"Record function id": 0, "Ev Idx": 5027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821311.871, "dur": 5.584,
    "args": {
      "External id": 2488229,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821337.966, "dur": 27.890,
    "args": {
      "External id": 2488230,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821342.628, "dur": 1.828,
    "args": {
      "External id": 2488231,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821346.990, "dur": 0.319,
    "args": {
      "External id": 2488232,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821348.833, "dur": 0.516,
    "args": {
      "External id": 2488233,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821351.560, "dur": 0.529,
    "args": {
      "External id": 2488234,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821353.374, "dur": 0.299,
    "args": {
      "External id": 2488235,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821354.901, "dur": 2.273,
    "args": {
      "External id": 2488236,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821358.642, "dur": 0.350,
    "args": {
      "External id": 2488237,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821360.327, "dur": 0.422,
    "args": {
      "External id": 2488238,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821362.596, "dur": 0.388,
    "args": {
      "External id": 2488239,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821377.077, "dur": 29.129,
    "args": {
      "External id": 2488240,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821436.277, "dur": 114.745,
    "args": {
      "External id": 2488241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821446.090, "dur": 3.130,
    "args": {
      "External id": 2488242,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821454.172, "dur": 9.923,
    "args": {
      "External id": 2488243,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821458.550, "dur": 5.145,
    "args": {
      "External id": 2488244,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821462.032, "dur": 0.481,
    "args": {
      "External id": 2488245,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821470.592, "dur": 26.220,
    "args": {
      "External id": 2488246,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821472.904, "dur": 0.415,
    "args": {
      "External id": 2488247,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821474.808, "dur": 0.318,
    "args": {
      "External id": 2488248,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821477.283, "dur": 2.054,
    "args": {
      "External id": 2488249,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821480.832, "dur": 0.391,
    "args": {
      "External id": 2488250,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821482.536, "dur": 0.285,
    "args": {
      "External id": 2488251,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821484.540, "dur": 0.271,
    "args": {
      "External id": 2488252,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821486.305, "dur": 0.289,
    "args": {
      "External id": 2488253,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821487.964, "dur": 0.365,
    "args": {
      "External id": 2488254,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821489.635, "dur": 0.385,
    "args": {
      "External id": 2488255,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821507.608, "dur": 18.806,
    "args": {
      "External id": 2488256,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821598.210, "dur": 116.092,
    "args": {
      "External id": 2488257,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821625.443, "dur": 85.580,
    "args": {
      "External id": 2488258,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5057, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821636.161, "dur": 70.698,
    "args": {
      "External id": 2488259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821730.105, "dur": 1.644,
    "args": {
      "External id": 2488260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5059, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821806.717, "dur": 1662.151,
    "args": {
      "External id": 2488261,"Sequence number": 24740430, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5060
    }
  },
  {
    "ph": "f", "id": 218, "pid": 1336760, "tid": 1381209, "ts": 1514657821806.717,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657821917.061, "dur": 145.064,
    "args": {
      "External id": 2488262,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822108.121, "dur": 42.407,
    "args": {
      "External id": 2488263,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822167.994, "dur": 56.817,
    "args": {
      "External id": 2488264,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822234.293, "dur": 32.823,
    "args": {
      "External id": 2488265,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822275.451, "dur": 44.344,
    "args": {
      "External id": 2488266,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822325.798, "dur": 27.516,
    "args": {
      "External id": 2488267,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822360.228, "dur": 42.348,
    "args": {
      "External id": 2488268,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822429.946, "dur": 24.448,
    "args": {
      "External id": 2488269,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822472.558, "dur": 31.224,
    "args": {
      "External id": 2488270,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822527.114, "dur": 36.906,
    "args": {
      "External id": 2488271,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822581.467, "dur": 15.156,
    "args": {
      "External id": 2488272,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822604.875, "dur": 34.924,
    "args": {
      "External id": 2488273,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822643.177, "dur": 34.707,
    "args": {
      "External id": 2488274,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822706.728, "dur": 165.024,
    "args": {
      "External id": 2488275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822781.591, "dur": 6.507,
    "args": {
      "External id": 2488276,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822789.866, "dur": 2.532,
    "args": {
      "External id": 2488277,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822909.720, "dur": 26.010,
    "args": {
      "External id": 2488278,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822948.143, "dur": 14.231,
    "args": {
      "External id": 2488279,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657822970.518, "dur": 74.063,
    "args": {
      "External id": 2488280,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823052.971, "dur": 39.006,
    "args": {
      "External id": 2488281,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823099.450, "dur": 20.792,
    "args": {
      "External id": 2488282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823126.243, "dur": 29.420,
    "args": {
      "External id": 2488283,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823161.525, "dur": 31.781,
    "args": {
      "External id": 2488284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823207.625, "dur": 34.625,
    "args": {
      "External id": 2488285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823264.536, "dur": 24.352,
    "args": {
      "External id": 2488286,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823313.056, "dur": 26.076,
    "args": {
      "External id": 2488287,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823362.010, "dur": 16.612,
    "args": {
      "External id": 2488288,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823395.342, "dur": 13.595,
    "args": {
      "External id": 2488289,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823425.434, "dur": 16.208,
    "args": {
      "External id": 2488290,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823513.403, "dur": 34.880,
    "args": {
      "External id": 2488291,"Record function id": 0, "Ev Idx": 5090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823516.636, "dur": 10.655,
    "args": {
      "External id": 2488292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823520.606, "dur": 5.842,
    "args": {
      "External id": 2488293,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823522.121, "dur": 4.235,
    "args": {
      "External id": 2488294,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823555.221, "dur": 6.744,
    "args": {
      "External id": 2488295,"Record function id": 0, "Ev Idx": 5094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823557.082, "dur": 4.386,
    "args": {
      "External id": 2488296,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823558.467, "dur": 2.236,
    "args": {
      "External id": 2488297,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823559.275, "dur": 1.339,
    "args": {
      "External id": 2488298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823565.583, "dur": 4.471,
    "args": {
      "External id": 2488299,"Record function id": 0, "Ev Idx": 5098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823566.649, "dur": 2.992,
    "args": {
      "External id": 2488300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823567.382, "dur": 1.831,
    "args": {
      "External id": 2488301,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823568.119, "dur": 1.015,
    "args": {
      "External id": 2488302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823573.252, "dur": 3.688,
    "args": {
      "External id": 2488303,"Record function id": 0, "Ev Idx": 5102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823574.540, "dur": 1.982,
    "args": {
      "External id": 2488304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823575.222, "dur": 0.899,
    "args": {
      "External id": 2488305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823575.500, "dur": 0.554,
    "args": {
      "External id": 2488306,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823580.007, "dur": 5.320,
    "args": {
      "External id": 2488307,"Record function id": 0, "Ev Idx": 5106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823581.116, "dur": 3.812,
    "args": {
      "External id": 2488308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823581.618, "dur": 2.874,
    "args": {
      "External id": 2488309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823581.887, "dur": 2.543,
    "args": {
      "External id": 2488310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823591.475, "dur": 3.670,
    "args": {
      "External id": 2488311,"Record function id": 0, "Ev Idx": 5110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823592.506, "dur": 2.253,
    "args": {
      "External id": 2488312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823592.942, "dur": 1.414,
    "args": {
      "External id": 2488313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823593.489, "dur": 0.803,
    "args": {
      "External id": 2488314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823598.349, "dur": 5.224,
    "args": {
      "External id": 2488315,"Record function id": 0, "Ev Idx": 5114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823599.534, "dur": 3.642,
    "args": {
      "External id": 2488316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823600.130, "dur": 2.531,
    "args": {
      "External id": 2488317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823601.950, "dur": 0.642,
    "args": {
      "External id": 2488318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823606.632, "dur": 25.527,
    "args": {
      "External id": 2488319,"Record function id": 0, "Ev Idx": 5118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823607.659, "dur": 24.081,
    "args": {
      "External id": 2488320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823630.255, "dur": 1.088,
    "args": {
      "External id": 2488321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823630.592, "dur": 0.687,
    "args": {
      "External id": 2488322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823635.256, "dur": 3.923,
    "args": {
      "External id": 2488323,"Record function id": 0, "Ev Idx": 5122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823636.753, "dur": 1.995,
    "args": {
      "External id": 2488324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823637.271, "dur": 1.067,
    "args": {
      "External id": 2488325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823637.756, "dur": 0.516,
    "args": {
      "External id": 2488326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823643.107, "dur": 35920.668,
    "args": {
      "External id": 2488327,"Record function id": 0, "Sequence number": 24740429, "Fwd thread id": 1, "Ev Idx": 5126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823644.906, "dur": 35908.604,
    "args": {
      "External id": 2488328,"Sequence number": 24740429, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5127
    }
  },
  {
    "ph": "f", "id": 219, "pid": 1336760, "tid": 1381209, "ts": 1514657823644.906,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823674.758, "dur": 37.346,
    "args": {
      "External id": 2488329,"Record function id": 0, "Ev Idx": 5128
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823719.311, "dur": 65.879,
    "args": {
      "External id": 2488330,"Record function id": 0, "Ev Idx": 5129
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.4)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823790.528, "dur": 35753.909,
    "args": {
      "External id": 2488331,"Record function id": 0, "Ev Idx": 5130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823883.215, "dur": 6.577,
    "args": {
      "External id": 2488332,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823899.685, "dur": 4.915,
    "args": {
      "External id": 2488333,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823917.571, "dur": 34790.135,
    "args": {
      "External id": 2488334,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823930.490, "dur": 34768.136,
    "args": {
      "External id": 2488335,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657823978.209, "dur": 52.204,
    "args": {
      "External id": 2488336,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657824038.068, "dur": 34617.531,
    "args": {
      "External id": 2488337,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657824040.500, "dur": 34614.501,
    "args": {
      "External id": 2488338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657824044.605, "dur": 11.664,
    "args": {
      "External id": 2488339,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657824059.578, "dur": 34591.996,
    "args": {
      "External id": 2488340,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657858796.284, "dur": 8.636,
    "args": {
      "External id": 2488341,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657858798.976, "dur": 5.619,
    "args": {
      "External id": 2488342,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657858834.041, "dur": 406.230,
    "args": {
      "External id": 2488343,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657858863.055, "dur": 371.556,
    "args": {
      "External id": 2488344,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5143, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657858874.355, "dur": 354.164,
    "args": {
      "External id": 2488345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859264.507, "dur": 2.316,
    "args": {
      "External id": 2488346,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5145, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859328.337, "dur": 6.415,
    "args": {
      "External id": 2488347,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859379.285, "dur": 1.393,
    "args": {
      "External id": 2488348,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859395.976, "dur": 1.572,
    "args": {
      "External id": 2488349,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859408.693, "dur": 2.778,
    "args": {
      "External id": 2488350,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859422.269, "dur": 1.117,
    "args": {
      "External id": 2488351,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859433.235, "dur": 1.033,
    "args": {
      "External id": 2488352,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859446.249, "dur": 1.160,
    "args": {
      "External id": 2488353,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859457.825, "dur": 3.253,
    "args": {
      "External id": 2488354,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859472.632, "dur": 0.697,
    "args": {
      "External id": 2488355,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859579.303, "dur": 2719.579,
    "args": {
      "External id": 2488356,"Record function id": 0, "Ev Idx": 5155
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859600.311, "dur": 1034.782,
    "args": {
      "External id": 2488357,"Record function id": 0, "Ev Idx": 5156
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859614.024, "dur": 312.862,
    "args": {
      "External id": 2488358,"Record function id": 0, "Ev Idx": 5157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859696.454, "dur": 5.039,
    "args": {
      "External id": 2488359,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859704.282, "dur": 1.079,
    "args": {
      "External id": 2488360,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859708.748, "dur": 1.146,
    "args": {
      "External id": 2488361,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859711.245, "dur": 0.909,
    "args": {
      "External id": 2488362,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859713.523, "dur": 0.722,
    "args": {
      "External id": 2488363,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859715.609, "dur": 2.762,
    "args": {
      "External id": 2488364,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859720.982, "dur": 0.948,
    "args": {
      "External id": 2488365,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859723.109, "dur": 0.982,
    "args": {
      "External id": 2488366,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859725.552, "dur": 1.507,
    "args": {
      "External id": 2488367,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859728.605, "dur": 1.627,
    "args": {
      "External id": 2488368,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859748.972, "dur": 150.599,
    "args": {
      "External id": 2488369,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859764.832, "dur": 130.279,
    "args": {
      "External id": 2488370,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859785.032, "dur": 12.034,
    "args": {
      "External id": 2488371,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859799.997, "dur": 67.528,
    "args": {
      "External id": 2488372,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859802.347, "dur": 64.775,
    "args": {
      "External id": 2488373,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859806.203, "dur": 6.073,
    "args": {
      "External id": 2488374,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657859813.880, "dur": 52.473,
    "args": {
      "External id": 2488375,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5174
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860053.147, "dur": 574.411,
    "args": {
      "External id": 2488376,"Record function id": 0, "Ev Idx": 5175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860074.289, "dur": 539.990,
    "args": {
      "External id": 2488377,"Record function id": 0, "Ev Idx": 5176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860142.347, "dur": 5.530,
    "args": {
      "External id": 2488378,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860168.871, "dur": 27.740,
    "args": {
      "External id": 2488379,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860173.565, "dur": 3.591,
    "args": {
      "External id": 2488380,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860179.518, "dur": 0.664,
    "args": {
      "External id": 2488381,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860181.591, "dur": 0.514,
    "args": {
      "External id": 2488382,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860183.728, "dur": 0.626,
    "args": {
      "External id": 2488383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860185.585, "dur": 0.453,
    "args": {
      "External id": 2488384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860186.793, "dur": 1.134,
    "args": {
      "External id": 2488385,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860188.806, "dur": 0.462,
    "args": {
      "External id": 2488386,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860189.998, "dur": 0.228,
    "args": {
      "External id": 2488387,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860191.618, "dur": 1.915,
    "args": {
      "External id": 2488388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860207.377, "dur": 33.158,
    "args": {
      "External id": 2488389,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860271.746, "dur": 94.522,
    "args": {
      "External id": 2488390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860281.228, "dur": 3.175,
    "args": {
      "External id": 2488391,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860289.606, "dur": 9.769,
    "args": {
      "External id": 2488392,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860293.913, "dur": 5.053,
    "args": {
      "External id": 2488393,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860297.152, "dur": 0.560,
    "args": {
      "External id": 2488394,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860305.605, "dur": 22.506,
    "args": {
      "External id": 2488395,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860307.237, "dur": 0.579,
    "args": {
      "External id": 2488396,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860309.181, "dur": 0.491,
    "args": {
      "External id": 2488397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860311.029, "dur": 0.605,
    "args": {
      "External id": 2488398,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860312.523, "dur": 0.461,
    "args": {
      "External id": 2488399,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860314.046, "dur": 0.418,
    "args": {
      "External id": 2488400,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860315.971, "dur": 1.860,
    "args": {
      "External id": 2488401,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860318.838, "dur": 0.374,
    "args": {
      "External id": 2488402,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860320.078, "dur": 0.373,
    "args": {
      "External id": 2488403,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860321.402, "dur": 0.271,
    "args": {
      "External id": 2488404,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860339.181, "dur": 19.643,
    "args": {
      "External id": 2488405,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860411.852, "dur": 111.287,
    "args": {
      "External id": 2488406,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860436.380, "dur": 83.420,
    "args": {
      "External id": 2488407,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5206, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860446.623, "dur": 69.373,
    "args": {
      "External id": 2488408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860557.666, "dur": 2.851,
    "args": {
      "External id": 2488409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5208, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860642.403, "dur": 1634.459,
    "args": {
      "External id": 2488410,"Sequence number": 24740428, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5209
    }
  },
  {
    "ph": "f", "id": 220, "pid": 1336760, "tid": 1381209, "ts": 1514657860642.403,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860759.931, "dur": 104.403,
    "args": {
      "External id": 2488411,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860908.822, "dur": 39.459,
    "args": {
      "External id": 2488412,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657860967.051, "dur": 92.346,
    "args": {
      "External id": 2488413,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861073.881, "dur": 34.350,
    "args": {
      "External id": 2488414,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861114.795, "dur": 44.048,
    "args": {
      "External id": 2488415,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861167.827, "dur": 26.588,
    "args": {
      "External id": 2488416,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861204.567, "dur": 40.664,
    "args": {
      "External id": 2488417,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861271.878, "dur": 23.410,
    "args": {
      "External id": 2488418,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861313.609, "dur": 26.868,
    "args": {
      "External id": 2488419,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861361.158, "dur": 17.683,
    "args": {
      "External id": 2488420,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861393.213, "dur": 14.454,
    "args": {
      "External id": 2488421,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861416.485, "dur": 29.240,
    "args": {
      "External id": 2488422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861448.894, "dur": 32.248,
    "args": {
      "External id": 2488423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861506.169, "dur": 193.621,
    "args": {
      "External id": 2488424,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861607.494, "dur": 6.947,
    "args": {
      "External id": 2488425,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861616.694, "dur": 2.547,
    "args": {
      "External id": 2488426,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861736.922, "dur": 24.211,
    "args": {
      "External id": 2488427,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861774.802, "dur": 13.231,
    "args": {
      "External id": 2488428,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861797.965, "dur": 37.732,
    "args": {
      "External id": 2488429,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861841.508, "dur": 34.000,
    "args": {
      "External id": 2488430,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861881.275, "dur": 19.665,
    "args": {
      "External id": 2488431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861905.075, "dur": 30.058,
    "args": {
      "External id": 2488432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861942.542, "dur": 19.833,
    "args": {
      "External id": 2488433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657861969.365, "dur": 71.396,
    "args": {
      "External id": 2488434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862077.671, "dur": 28.475,
    "args": {
      "External id": 2488435,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862128.285, "dur": 23.356,
    "args": {
      "External id": 2488436,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862166.943, "dur": 19.262,
    "args": {
      "External id": 2488437,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862203.079, "dur": 13.477,
    "args": {
      "External id": 2488438,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862229.220, "dur": 19.576,
    "args": {
      "External id": 2488439,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862321.471, "dur": 17.746,
    "args": {
      "External id": 2488440,"Record function id": 0, "Ev Idx": 5239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862324.775, "dur": 13.513,
    "args": {
      "External id": 2488441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862328.932, "dur": 8.432,
    "args": {
      "External id": 2488442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862330.679, "dur": 6.553,
    "args": {
      "External id": 2488443,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862343.102, "dur": 4.590,
    "args": {
      "External id": 2488444,"Record function id": 0, "Ev Idx": 5243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862344.235, "dur": 3.044,
    "args": {
      "External id": 2488445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862345.015, "dur": 1.816,
    "args": {
      "External id": 2488446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862345.852, "dur": 0.914,
    "args": {
      "External id": 2488447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862354.017, "dur": 4.031,
    "args": {
      "External id": 2488448,"Record function id": 0, "Ev Idx": 5247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862355.231, "dur": 2.411,
    "args": {
      "External id": 2488449,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862355.907, "dur": 1.311,
    "args": {
      "External id": 2488450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862356.420, "dur": 0.735,
    "args": {
      "External id": 2488451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862361.195, "dur": 3.569,
    "args": {
      "External id": 2488452,"Record function id": 0, "Ev Idx": 5251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862362.194, "dur": 2.174,
    "args": {
      "External id": 2488453,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862362.738, "dur": 1.215,
    "args": {
      "External id": 2488454,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862363.235, "dur": 0.656,
    "args": {
      "External id": 2488455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862367.901, "dur": 3.339,
    "args": {
      "External id": 2488456,"Record function id": 0, "Ev Idx": 5255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862368.862, "dur": 1.980,
    "args": {
      "External id": 2488457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862369.403, "dur": 1.023,
    "args": {
      "External id": 2488458,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862369.793, "dur": 0.564,
    "args": {
      "External id": 2488459,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862374.339, "dur": 3.783,
    "args": {
      "External id": 2488460,"Record function id": 0, "Ev Idx": 5259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862375.274, "dur": 2.429,
    "args": {
      "External id": 2488461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862375.784, "dur": 1.473,
    "args": {
      "External id": 2488462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862376.257, "dur": 0.907,
    "args": {
      "External id": 2488463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862381.444, "dur": 3.458,
    "args": {
      "External id": 2488464,"Record function id": 0, "Ev Idx": 5263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862382.547, "dur": 1.908,
    "args": {
      "External id": 2488465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862383.045, "dur": 0.998,
    "args": {
      "External id": 2488466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862383.326, "dur": 0.629,
    "args": {
      "External id": 2488467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862388.031, "dur": 8.475,
    "args": {
      "External id": 2488468,"Record function id": 0, "Ev Idx": 5267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862388.964, "dur": 7.093,
    "args": {
      "External id": 2488469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862393.106, "dur": 2.525,
    "args": {
      "External id": 2488470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862393.432, "dur": 2.117,
    "args": {
      "External id": 2488471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862399.772, "dur": 4.509,
    "args": {
      "External id": 2488472,"Record function id": 0, "Ev Idx": 5271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862400.838, "dur": 3.018,
    "args": {
      "External id": 2488473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862401.387, "dur": 2.037,
    "args": {
      "External id": 2488474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862402.670, "dur": 0.645,
    "args": {
      "External id": 2488475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862407.988, "dur": 36850.462,
    "args": {
      "External id": 2488476,"Record function id": 0, "Sequence number": 24740427, "Fwd thread id": 1, "Ev Idx": 5275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862409.356, "dur": 36840.147,
    "args": {
      "External id": 2488477,"Sequence number": 24740427, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5276
    }
  },
  {
    "ph": "f", "id": 221, "pid": 1336760, "tid": 1381209, "ts": 1514657862409.356,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862436.332, "dur": 35.956,
    "args": {
      "External id": 2488478,"Record function id": 0, "Ev Idx": 5277
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862479.823, "dur": 83.798,
    "args": {
      "External id": 2488479,"Record function id": 0, "Ev Idx": 5278
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.3)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862571.012, "dur": 36670.469,
    "args": {
      "External id": 2488480,"Record function id": 0, "Ev Idx": 5279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862660.129, "dur": 6.680,
    "args": {
      "External id": 2488481,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862676.488, "dur": 5.454,
    "args": {
      "External id": 2488482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862695.661, "dur": 35711.615,
    "args": {
      "External id": 2488483,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862708.896, "dur": 35688.240,
    "args": {
      "External id": 2488484,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862754.877, "dur": 13.234,
    "args": {
      "External id": 2488485,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862774.179, "dur": 35581.889,
    "args": {
      "External id": 2488486,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862776.837, "dur": 35578.213,
    "args": {
      "External id": 2488487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862780.935, "dur": 5.179,
    "args": {
      "External id": 2488488,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657862787.715, "dur": 35563.139,
    "args": {
      "External id": 2488489,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898504.575, "dur": 10.469,
    "args": {
      "External id": 2488490,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898507.665, "dur": 7.019,
    "args": {
      "External id": 2488491,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898557.564, "dur": 355.544,
    "args": {
      "External id": 2488492,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898585.115, "dur": 323.326,
    "args": {
      "External id": 2488493,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5292, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898599.023, "dur": 304.059,
    "args": {
      "External id": 2488494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657898930.864, "dur": 2.233,
    "args": {
      "External id": 2488495,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5294, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899036.728, "dur": 7.237,
    "args": {
      "External id": 2488496,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899088.254, "dur": 1.484,
    "args": {
      "External id": 2488497,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899104.898, "dur": 3.022,
    "args": {
      "External id": 2488498,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899119.211, "dur": 0.917,
    "args": {
      "External id": 2488499,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899131.370, "dur": 0.766,
    "args": {
      "External id": 2488500,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899141.483, "dur": 1.101,
    "args": {
      "External id": 2488501,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899152.337, "dur": 3.223,
    "args": {
      "External id": 2488502,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899170.557, "dur": 1.562,
    "args": {
      "External id": 2488503,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899181.618, "dur": 0.810,
    "args": {
      "External id": 2488504,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899273.463, "dur": 2783.538,
    "args": {
      "External id": 2488505,"Record function id": 0, "Ev Idx": 5304
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899293.005, "dur": 1053.924,
    "args": {
      "External id": 2488506,"Record function id": 0, "Ev Idx": 5305
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899307.632, "dur": 338.362,
    "args": {
      "External id": 2488507,"Record function id": 0, "Ev Idx": 5306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899388.571, "dur": 4.197,
    "args": {
      "External id": 2488508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899395.880, "dur": 1.288,
    "args": {
      "External id": 2488509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899399.101, "dur": 0.958,
    "args": {
      "External id": 2488510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899401.669, "dur": 2.682,
    "args": {
      "External id": 2488511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899407.056, "dur": 1.105,
    "args": {
      "External id": 2488512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899409.363, "dur": 0.871,
    "args": {
      "External id": 2488513,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899411.775, "dur": 1.142,
    "args": {
      "External id": 2488514,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899414.586, "dur": 0.576,
    "args": {
      "External id": 2488515,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899417.904, "dur": 1.054,
    "args": {
      "External id": 2488516,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899420.214, "dur": 1.101,
    "args": {
      "External id": 2488517,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899439.400, "dur": 175.991,
    "args": {
      "External id": 2488518,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899454.203, "dur": 156.575,
    "args": {
      "External id": 2488519,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899473.223, "dur": 12.778,
    "args": {
      "External id": 2488520,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899489.074, "dur": 93.030,
    "args": {
      "External id": 2488521,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899491.889, "dur": 89.685,
    "args": {
      "External id": 2488522,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899495.646, "dur": 8.400,
    "args": {
      "External id": 2488523,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899507.373, "dur": 73.320,
    "args": {
      "External id": 2488524,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5323
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899733.731, "dur": 605.497,
    "args": {
      "External id": 2488525,"Record function id": 0, "Ev Idx": 5324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899755.218, "dur": 572.771,
    "args": {
      "External id": 2488526,"Record function id": 0, "Ev Idx": 5325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899817.724, "dur": 5.291,
    "args": {
      "External id": 2488527,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899846.425, "dur": 27.666,
    "args": {
      "External id": 2488528,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899851.512, "dur": 1.820,
    "args": {
      "External id": 2488529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899855.652, "dur": 0.352,
    "args": {
      "External id": 2488530,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899857.102, "dur": 0.310,
    "args": {
      "External id": 2488531,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899859.336, "dur": 0.487,
    "args": {
      "External id": 2488532,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899861.698, "dur": 0.493,
    "args": {
      "External id": 2488533,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899863.319, "dur": 0.565,
    "args": {
      "External id": 2488534,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899865.399, "dur": 2.304,
    "args": {
      "External id": 2488535,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899869.399, "dur": 0.433,
    "args": {
      "External id": 2488536,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899870.917, "dur": 0.425,
    "args": {
      "External id": 2488537,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899884.596, "dur": 32.099,
    "args": {
      "External id": 2488538,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899946.452, "dur": 154.436,
    "args": {
      "External id": 2488539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899955.901, "dur": 2.805,
    "args": {
      "External id": 2488540,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899963.903, "dur": 10.330,
    "args": {
      "External id": 2488541,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899968.306, "dur": 5.518,
    "args": {
      "External id": 2488542,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657899972.148, "dur": 0.539,
    "args": {
      "External id": 2488543,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900029.607, "dur": 29.252,
    "args": {
      "External id": 2488544,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900032.217, "dur": 0.591,
    "args": {
      "External id": 2488545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900034.514, "dur": 0.654,
    "args": {
      "External id": 2488546,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900036.576, "dur": 0.320,
    "args": {
      "External id": 2488547,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900038.394, "dur": 2.140,
    "args": {
      "External id": 2488548,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900042.077, "dur": 0.383,
    "args": {
      "External id": 2488549,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900043.476, "dur": 0.507,
    "args": {
      "External id": 2488550,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900047.596, "dur": 0.467,
    "args": {
      "External id": 2488551,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900049.376, "dur": 0.735,
    "args": {
      "External id": 2488552,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900051.369, "dur": 0.500,
    "args": {
      "External id": 2488553,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900071.179, "dur": 22.191,
    "args": {
      "External id": 2488554,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900145.645, "dur": 115.452,
    "args": {
      "External id": 2488555,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900169.515, "dur": 88.373,
    "args": {
      "External id": 2488556,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5355, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900179.549, "dur": 74.554,
    "args": {
      "External id": 2488557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900275.571, "dur": 2.072,
    "args": {
      "External id": 2488558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5357, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900354.252, "dur": 1678.382,
    "args": {
      "External id": 2488559,"Sequence number": 24740426, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5358
    }
  },
  {
    "ph": "f", "id": 222, "pid": 1336760, "tid": 1381209, "ts": 1514657900354.252,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900465.359, "dur": 122.634,
    "args": {
      "External id": 2488560,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900633.045, "dur": 43.525,
    "args": {
      "External id": 2488561,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900693.664, "dur": 55.424,
    "args": {
      "External id": 2488562,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900759.241, "dur": 32.165,
    "args": {
      "External id": 2488563,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900797.939, "dur": 44.351,
    "args": {
      "External id": 2488564,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900850.978, "dur": 27.750,
    "args": {
      "External id": 2488565,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900885.364, "dur": 41.215,
    "args": {
      "External id": 2488566,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657900951.210, "dur": 22.066,
    "args": {
      "External id": 2488567,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901031.982, "dur": 32.253,
    "args": {
      "External id": 2488568,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901087.871, "dur": 18.674,
    "args": {
      "External id": 2488569,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901120.070, "dur": 14.827,
    "args": {
      "External id": 2488570,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901146.242, "dur": 34.116,
    "args": {
      "External id": 2488571,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901183.911, "dur": 33.618,
    "args": {
      "External id": 2488572,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901247.619, "dur": 173.756,
    "args": {
      "External id": 2488573,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901326.058, "dur": 5.531,
    "args": {
      "External id": 2488574,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901333.773, "dur": 2.317,
    "args": {
      "External id": 2488575,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901461.653, "dur": 24.540,
    "args": {
      "External id": 2488576,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901497.155, "dur": 13.761,
    "args": {
      "External id": 2488577,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901524.284, "dur": 55.955,
    "args": {
      "External id": 2488578,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901589.487, "dur": 36.361,
    "args": {
      "External id": 2488579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901635.948, "dur": 20.789,
    "args": {
      "External id": 2488580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901661.525, "dur": 32.314,
    "args": {
      "External id": 2488581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901717.941, "dur": 29.780,
    "args": {
      "External id": 2488582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901756.564, "dur": 30.997,
    "args": {
      "External id": 2488583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901806.688, "dur": 23.448,
    "args": {
      "External id": 2488584,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901848.135, "dur": 29.140,
    "args": {
      "External id": 2488585,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901891.931, "dur": 16.045,
    "args": {
      "External id": 2488586,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901921.829, "dur": 17.043,
    "args": {
      "External id": 2488587,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657901950.738, "dur": 14.846,
    "args": {
      "External id": 2488588,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902079.864, "dur": 15.748,
    "args": {
      "External id": 2488589,"Record function id": 0, "Ev Idx": 5388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902083.332, "dur": 11.256,
    "args": {
      "External id": 2488590,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902087.464, "dur": 6.280,
    "args": {
      "External id": 2488591,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902089.254, "dur": 4.335,
    "args": {
      "External id": 2488592,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902099.581, "dur": 5.373,
    "args": {
      "External id": 2488593,"Record function id": 0, "Ev Idx": 5392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902101.073, "dur": 3.441,
    "args": {
      "External id": 2488594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902101.985, "dur": 2.030,
    "args": {
      "External id": 2488595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902102.888, "dur": 1.063,
    "args": {
      "External id": 2488596,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902108.392, "dur": 4.500,
    "args": {
      "External id": 2488597,"Record function id": 0, "Ev Idx": 5396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902109.458, "dur": 3.029,
    "args": {
      "External id": 2488598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902110.246, "dur": 1.823,
    "args": {
      "External id": 2488599,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902110.937, "dur": 1.044,
    "args": {
      "External id": 2488600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902116.300, "dur": 6.438,
    "args": {
      "External id": 2488601,"Record function id": 0, "Ev Idx": 5400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902117.808, "dur": 4.504,
    "args": {
      "External id": 2488602,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902118.601, "dur": 3.286,
    "args": {
      "External id": 2488603,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902119.094, "dur": 2.731,
    "args": {
      "External id": 2488604,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902125.981, "dur": 3.777,
    "args": {
      "External id": 2488605,"Record function id": 0, "Ev Idx": 5404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902127.213, "dur": 2.132,
    "args": {
      "External id": 2488606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902127.777, "dur": 1.117,
    "args": {
      "External id": 2488607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902128.087, "dur": 0.745,
    "args": {
      "External id": 2488608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902133.040, "dur": 3.993,
    "args": {
      "External id": 2488609,"Record function id": 0, "Ev Idx": 5408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902134.054, "dur": 2.526,
    "args": {
      "External id": 2488610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902134.687, "dur": 1.426,
    "args": {
      "External id": 2488611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902135.170, "dur": 0.833,
    "args": {
      "External id": 2488612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902140.598, "dur": 3.867,
    "args": {
      "External id": 2488613,"Record function id": 0, "Ev Idx": 5412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902141.712, "dur": 2.346,
    "args": {
      "External id": 2488614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902142.226, "dur": 1.386,
    "args": {
      "External id": 2488615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902142.625, "dur": 0.911,
    "args": {
      "External id": 2488616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902147.624, "dur": 3.799,
    "args": {
      "External id": 2488617,"Record function id": 0, "Ev Idx": 5416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902148.663, "dur": 2.348,
    "args": {
      "External id": 2488618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902149.367, "dur": 1.213,
    "args": {
      "External id": 2488619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902149.670, "dur": 0.814,
    "args": {
      "External id": 2488620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902154.579, "dur": 4.085,
    "args": {
      "External id": 2488621,"Record function id": 0, "Ev Idx": 5420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902155.770, "dur": 2.500,
    "args": {
      "External id": 2488622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902156.517, "dur": 1.348,
    "args": {
      "External id": 2488623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902157.192, "dur": 0.560,
    "args": {
      "External id": 2488624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902162.506, "dur": 37294.329,
    "args": {
      "External id": 2488625,"Record function id": 0, "Sequence number": 24740425, "Fwd thread id": 1, "Ev Idx": 5424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902164.379, "dur": 37284.151,
    "args": {
      "External id": 2488626,"Sequence number": 24740425, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5425
    }
  },
  {
    "ph": "f", "id": 223, "pid": 1336760, "tid": 1381209, "ts": 1514657902164.379,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902194.640, "dur": 36.021,
    "args": {
      "External id": 2488627,"Record function id": 0, "Ev Idx": 5426
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902238.902, "dur": 68.252,
    "args": {
      "External id": 2488628,"Record function id": 0, "Ev Idx": 5427
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.2)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902312.519, "dur": 37127.591,
    "args": {
      "External id": 2488629,"Record function id": 0, "Ev Idx": 5428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902407.008, "dur": 6.120,
    "args": {
      "External id": 2488630,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902422.816, "dur": 4.934,
    "args": {
      "External id": 2488631,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902442.281, "dur": 36169.589,
    "args": {
      "External id": 2488632,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902456.027, "dur": 36146.787,
    "args": {
      "External id": 2488633,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902507.304, "dur": 13.515,
    "args": {
      "External id": 2488634,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902526.891, "dur": 36036.097,
    "args": {
      "External id": 2488635,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902546.035, "dur": 36016.222,
    "args": {
      "External id": 2488636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902550.193, "dur": 8.006,
    "args": {
      "External id": 2488637,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657902560.252, "dur": 35998.534,
    "args": {
      "External id": 2488638,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657938701.318, "dur": 10.114,
    "args": {
      "External id": 2488639,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657938704.134, "dur": 6.945,
    "args": {
      "External id": 2488640,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657938739.107, "dur": 412.989,
    "args": {
      "External id": 2488641,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657938769.603, "dur": 377.134,
    "args": {
      "External id": 2488642,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5441, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657938781.223, "dur": 359.219,
    "args": {
      "External id": 2488643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939172.108, "dur": 2.302,
    "args": {
      "External id": 2488644,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5443, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939239.354, "dur": 6.675,
    "args": {
      "External id": 2488645,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939289.626, "dur": 1.365,
    "args": {
      "External id": 2488646,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939306.181, "dur": 1.164,
    "args": {
      "External id": 2488647,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939318.529, "dur": 2.892,
    "args": {
      "External id": 2488648,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939332.230, "dur": 0.839,
    "args": {
      "External id": 2488649,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939343.786, "dur": 1.052,
    "args": {
      "External id": 2488650,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939354.478, "dur": 0.940,
    "args": {
      "External id": 2488651,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939365.971, "dur": 2.760,
    "args": {
      "External id": 2488652,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939379.673, "dur": 0.836,
    "args": {
      "External id": 2488653,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939471.284, "dur": 2780.984,
    "args": {
      "External id": 2488654,"Record function id": 0, "Ev Idx": 5453
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939491.840, "dur": 1083.934,
    "args": {
      "External id": 2488655,"Record function id": 0, "Ev Idx": 5454
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939508.316, "dur": 348.677,
    "args": {
      "External id": 2488656,"Record function id": 0, "Ev Idx": 5455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939620.848, "dur": 4.837,
    "args": {
      "External id": 2488657,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939629.256, "dur": 4.229,
    "args": {
      "External id": 2488658,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939635.122, "dur": 1.105,
    "args": {
      "External id": 2488659,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939638.101, "dur": 1.135,
    "args": {
      "External id": 2488660,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939640.906, "dur": 0.802,
    "args": {
      "External id": 2488661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939643.036, "dur": 0.778,
    "args": {
      "External id": 2488662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939645.418, "dur": 2.675,
    "args": {
      "External id": 2488663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939649.806, "dur": 1.160,
    "args": {
      "External id": 2488664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939652.312, "dur": 1.292,
    "args": {
      "External id": 2488665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939655.330, "dur": 1.346,
    "args": {
      "External id": 2488666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939676.480, "dur": 152.038,
    "args": {
      "External id": 2488667,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939692.423, "dur": 131.942,
    "args": {
      "External id": 2488668,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939711.323, "dur": 12.561,
    "args": {
      "External id": 2488669,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939727.080, "dur": 69.579,
    "args": {
      "External id": 2488670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939729.918, "dur": 66.352,
    "args": {
      "External id": 2488671,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939734.157, "dur": 6.484,
    "args": {
      "External id": 2488672,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939742.142, "dur": 53.409,
    "args": {
      "External id": 2488673,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5472
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939948.583, "dur": 619.512,
    "args": {
      "External id": 2488674,"Record function id": 0, "Ev Idx": 5473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657939969.645, "dur": 583.977,
    "args": {
      "External id": 2488675,"Record function id": 0, "Ev Idx": 5474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940076.629, "dur": 6.088,
    "args": {
      "External id": 2488676,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940099.297, "dur": 27.033,
    "args": {
      "External id": 2488677,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940104.189, "dur": 1.843,
    "args": {
      "External id": 2488678,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940108.046, "dur": 0.400,
    "args": {
      "External id": 2488679,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940110.032, "dur": 2.106,
    "args": {
      "External id": 2488680,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940113.691, "dur": 0.389,
    "args": {
      "External id": 2488681,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940115.535, "dur": 0.255,
    "args": {
      "External id": 2488682,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940117.673, "dur": 0.599,
    "args": {
      "External id": 2488683,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940119.598, "dur": 0.355,
    "args": {
      "External id": 2488684,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940121.417, "dur": 0.324,
    "args": {
      "External id": 2488685,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940123.030, "dur": 0.365,
    "args": {
      "External id": 2488686,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940137.778, "dur": 34.941,
    "args": {
      "External id": 2488687,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940203.630, "dur": 104.898,
    "args": {
      "External id": 2488688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940213.625, "dur": 2.900,
    "args": {
      "External id": 2488689,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940221.536, "dur": 11.882,
    "args": {
      "External id": 2488690,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940225.815, "dur": 7.183,
    "args": {
      "External id": 2488691,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940229.196, "dur": 2.610,
    "args": {
      "External id": 2488692,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940242.949, "dur": 24.609,
    "args": {
      "External id": 2488693,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940244.671, "dur": 0.557,
    "args": {
      "External id": 2488694,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940246.568, "dur": 0.297,
    "args": {
      "External id": 2488695,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940248.187, "dur": 0.363,
    "args": {
      "External id": 2488696,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940250.178, "dur": 0.428,
    "args": {
      "External id": 2488697,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940252.176, "dur": 0.367,
    "args": {
      "External id": 2488698,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940253.425, "dur": 0.308,
    "args": {
      "External id": 2488699,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940254.984, "dur": 0.312,
    "args": {
      "External id": 2488700,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940257.158, "dur": 2.043,
    "args": {
      "External id": 2488701,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940260.203, "dur": 0.629,
    "args": {
      "External id": 2488702,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940278.420, "dur": 22.197,
    "args": {
      "External id": 2488703,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940351.552, "dur": 116.088,
    "args": {
      "External id": 2488704,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940376.905, "dur": 87.555,
    "args": {
      "External id": 2488705,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5504, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940386.725, "dur": 73.589,
    "args": {
      "External id": 2488706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940484.966, "dur": 1.920,
    "args": {
      "External id": 2488707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5506, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940584.051, "dur": 1647.178,
    "args": {
      "External id": 2488708,"Sequence number": 24740424, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5507
    }
  },
  {
    "ph": "f", "id": 224, "pid": 1336760, "tid": 1381209, "ts": 1514657940584.051,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940700.965, "dur": 107.732,
    "args": {
      "External id": 2488709,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940851.622, "dur": 40.306,
    "args": {
      "External id": 2488710,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940907.155, "dur": 48.238,
    "args": {
      "External id": 2488711,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657940966.202, "dur": 74.056,
    "args": {
      "External id": 2488712,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941050.958, "dur": 48.474,
    "args": {
      "External id": 2488713,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941106.307, "dur": 26.689,
    "args": {
      "External id": 2488714,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941140.427, "dur": 42.011,
    "args": {
      "External id": 2488715,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941210.726, "dur": 23.805,
    "args": {
      "External id": 2488716,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941254.291, "dur": 31.115,
    "args": {
      "External id": 2488717,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941305.853, "dur": 18.987,
    "args": {
      "External id": 2488718,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941338.076, "dur": 15.483,
    "args": {
      "External id": 2488719,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941363.252, "dur": 29.680,
    "args": {
      "External id": 2488720,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941396.177, "dur": 32.158,
    "args": {
      "External id": 2488721,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941457.844, "dur": 182.666,
    "args": {
      "External id": 2488722,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941546.771, "dur": 6.676,
    "args": {
      "External id": 2488723,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941555.648, "dur": 2.703,
    "args": {
      "External id": 2488724,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941672.952, "dur": 23.227,
    "args": {
      "External id": 2488725,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941706.878, "dur": 14.418,
    "args": {
      "External id": 2488726,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941729.936, "dur": 39.657,
    "args": {
      "External id": 2488727,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941775.649, "dur": 35.180,
    "args": {
      "External id": 2488728,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941818.870, "dur": 20.939,
    "args": {
      "External id": 2488729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941845.282, "dur": 29.309,
    "args": {
      "External id": 2488730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941880.871, "dur": 33.865,
    "args": {
      "External id": 2488731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657941928.240, "dur": 38.857,
    "args": {
      "External id": 2488732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942021.465, "dur": 26.843,
    "args": {
      "External id": 2488733,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942077.522, "dur": 25.505,
    "args": {
      "External id": 2488734,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942121.960, "dur": 17.149,
    "args": {
      "External id": 2488735,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942156.344, "dur": 14.853,
    "args": {
      "External id": 2488736,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942188.341, "dur": 15.287,
    "args": {
      "External id": 2488737,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942274.033, "dur": 17.112,
    "args": {
      "External id": 2488738,"Record function id": 0, "Ev Idx": 5537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942277.424, "dur": 12.840,
    "args": {
      "External id": 2488739,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942281.404, "dur": 7.863,
    "args": {
      "External id": 2488740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942282.861, "dur": 6.237,
    "args": {
      "External id": 2488741,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942298.109, "dur": 4.963,
    "args": {
      "External id": 2488742,"Record function id": 0, "Ev Idx": 5541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942299.288, "dur": 3.378,
    "args": {
      "External id": 2488743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942300.087, "dur": 2.109,
    "args": {
      "External id": 2488744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942300.824, "dur": 1.273,
    "args": {
      "External id": 2488745,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942306.469, "dur": 3.925,
    "args": {
      "External id": 2488746,"Record function id": 0, "Ev Idx": 5545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942307.550, "dur": 2.461,
    "args": {
      "External id": 2488747,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942308.248, "dur": 1.322,
    "args": {
      "External id": 2488748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942308.764, "dur": 0.740,
    "args": {
      "External id": 2488749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942313.678, "dur": 3.528,
    "args": {
      "External id": 2488750,"Record function id": 0, "Ev Idx": 5549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942314.744, "dur": 2.027,
    "args": {
      "External id": 2488751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942315.287, "dur": 1.060,
    "args": {
      "External id": 2488752,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942315.584, "dur": 0.697,
    "args": {
      "External id": 2488753,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942320.365, "dur": 4.307,
    "args": {
      "External id": 2488754,"Record function id": 0, "Ev Idx": 5553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942321.652, "dur": 2.633,
    "args": {
      "External id": 2488755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942322.382, "dur": 1.484,
    "args": {
      "External id": 2488756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942323.091, "dur": 0.710,
    "args": {
      "External id": 2488757,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942327.899, "dur": 3.771,
    "args": {
      "External id": 2488758,"Record function id": 0, "Ev Idx": 5557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942328.869, "dur": 2.369,
    "args": {
      "External id": 2488759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942329.495, "dur": 1.203,
    "args": {
      "External id": 2488760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942329.930, "dur": 0.667,
    "args": {
      "External id": 2488761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942335.060, "dur": 4.126,
    "args": {
      "External id": 2488762,"Record function id": 0, "Ev Idx": 5561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942336.406, "dur": 2.382,
    "args": {
      "External id": 2488763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942336.988, "dur": 1.373,
    "args": {
      "External id": 2488764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942337.593, "dur": 0.689,
    "args": {
      "External id": 2488765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942342.388, "dur": 4.992,
    "args": {
      "External id": 2488766,"Record function id": 0, "Ev Idx": 5565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942343.368, "dur": 3.602,
    "args": {
      "External id": 2488767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942343.921, "dur": 2.651,
    "args": {
      "External id": 2488768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942344.354, "dur": 2.133,
    "args": {
      "External id": 2488769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942354.206, "dur": 4.068,
    "args": {
      "External id": 2488770,"Record function id": 0, "Ev Idx": 5569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942355.403, "dur": 2.453,
    "args": {
      "External id": 2488771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942356.123, "dur": 1.338,
    "args": {
      "External id": 2488772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942356.705, "dur": 0.645,
    "args": {
      "External id": 2488773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942361.918, "dur": 36753.154,
    "args": {
      "External id": 2488774,"Record function id": 0, "Sequence number": 24740423, "Fwd thread id": 1, "Ev Idx": 5573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942363.148, "dur": 36743.160,
    "args": {
      "External id": 2488775,"Sequence number": 24740423, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5574
    }
  },
  {
    "ph": "f", "id": 225, "pid": 1336760, "tid": 1381209, "ts": 1514657942363.148,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942390.407, "dur": 39.453,
    "args": {
      "External id": 2488776,"Record function id": 0, "Ev Idx": 5575
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942437.382, "dur": 69.233,
    "args": {
      "External id": 2488777,"Record function id": 0, "Ev Idx": 5576
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.1)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942512.400, "dur": 36585.789,
    "args": {
      "External id": 2488778,"Record function id": 0, "Ev Idx": 5577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942622.014, "dur": 6.893,
    "args": {
      "External id": 2488779,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942639.218, "dur": 5.152,
    "args": {
      "External id": 2488780,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942658.714, "dur": 35621.390,
    "args": {
      "External id": 2488781,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942672.645, "dur": 35597.241,
    "args": {
      "External id": 2488782,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942719.429, "dur": 13.166,
    "args": {
      "External id": 2488783,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942738.525, "dur": 35493.249,
    "args": {
      "External id": 2488784,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942740.862, "dur": 35490.143,
    "args": {
      "External id": 2488785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942745.067, "dur": 4.878,
    "args": {
      "External id": 2488786,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657942751.606, "dur": 35475.238,
    "args": {
      "External id": 2488787,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978376.935, "dur": 10.383,
    "args": {
      "External id": 2488788,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978381.087, "dur": 5.943,
    "args": {
      "External id": 2488789,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978416.221, "dur": 359.098,
    "args": {
      "External id": 2488790,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978442.877, "dur": 327.657,
    "args": {
      "External id": 2488791,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5590, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978454.607, "dur": 309.794,
    "args": {
      "External id": 2488792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978797.323, "dur": 2.417,
    "args": {
      "External id": 2488793,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5592, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978859.682, "dur": 6.694,
    "args": {
      "External id": 2488794,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978909.944, "dur": 1.672,
    "args": {
      "External id": 2488795,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978927.387, "dur": 2.771,
    "args": {
      "External id": 2488796,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978941.502, "dur": 1.259,
    "args": {
      "External id": 2488797,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978953.537, "dur": 0.897,
    "args": {
      "External id": 2488798,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978964.635, "dur": 1.336,
    "args": {
      "External id": 2488799,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657978975.652, "dur": 2.944,
    "args": {
      "External id": 2488800,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979026.361, "dur": 2.109,
    "args": {
      "External id": 2488801,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979038.430, "dur": 0.895,
    "args": {
      "External id": 2488802,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979130.228, "dur": 2159.261,
    "args": {
      "External id": 2488803,"Record function id": 0, "Ev Idx": 5602
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979153.642, "dur": 458.011,
    "args": {
      "External id": 2488804,"Record function id": 0, "Ev Idx": 5603
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979167.703, "dur": 322.669,
    "args": {
      "External id": 2488805,"Record function id": 0, "Ev Idx": 5604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979258.455, "dur": 4.242,
    "args": {
      "External id": 2488806,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979265.793, "dur": 1.317,
    "args": {
      "External id": 2488807,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979269.085, "dur": 0.923,
    "args": {
      "External id": 2488808,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979272.080, "dur": 1.196,
    "args": {
      "External id": 2488809,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979274.512, "dur": 2.584,
    "args": {
      "External id": 2488810,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979278.514, "dur": 0.777,
    "args": {
      "External id": 2488811,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979281.643, "dur": 0.793,
    "args": {
      "External id": 2488812,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979283.983, "dur": 0.784,
    "args": {
      "External id": 2488813,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979286.110, "dur": 1.559,
    "args": {
      "External id": 2488814,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979289.357, "dur": 1.275,
    "args": {
      "External id": 2488815,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979310.488, "dur": 149.599,
    "args": {
      "External id": 2488816,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979328.005, "dur": 127.327,
    "args": {
      "External id": 2488817,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979345.418, "dur": 12.297,
    "args": {
      "External id": 2488818,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979360.819, "dur": 67.291,
    "args": {
      "External id": 2488819,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979363.319, "dur": 64.457,
    "args": {
      "External id": 2488820,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979367.200, "dur": 6.111,
    "args": {
      "External id": 2488821,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979374.872, "dur": 52.353,
    "args": {
      "External id": 2488822,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979621.238, "dur": 1645.381,
    "args": {
      "External id": 2488823,"Sequence number": 24740422, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5622
    }
  },
  {
    "ph": "f", "id": 226, "pid": 1336760, "tid": 1381209, "ts": 1514657979621.238,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979740.492, "dur": 107.638,
    "args": {
      "External id": 2488824,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979884.722, "dur": 45.099,
    "args": {
      "External id": 2488825,"kernel_hash": "cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/xo/cxonx5mk7e7epy33myj3jjhyzcg75j44argg6g4lybldimdgtudf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657979949.477, "dur": 92.788,
    "args": {
      "External id": 2488826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980056.656, "dur": 37.785,
    "args": {
      "External id": 2488827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980104.099, "dur": 45.180,
    "args": {
      "External id": 2488828,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980156.390, "dur": 27.842,
    "args": {
      "External id": 2488829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980191.543, "dur": 41.786,
    "args": {
      "External id": 2488830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980257.398, "dur": 22.202,
    "args": {
      "External id": 2488831,"kernel_hash": "czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/zp/czpg7m5cvhtnsrbedscn3z34ms26dcj4yxfddqcc2lsxt5vfm3gr.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980299.773, "dur": 28.058,
    "args": {
      "External id": 2488832,"kernel_hash": "c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/47/c47krgrbnxrqlxc37sur5dq2rv5rcz5jecidebny2djfdualz2i5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980374.457, "dur": 19.616,
    "args": {
      "External id": 2488833,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980408.280, "dur": 14.692,
    "args": {
      "External id": 2488834,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980434.365, "dur": 30.024,
    "args": {
      "External id": 2488835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980467.499, "dur": 31.985,
    "args": {
      "External id": 2488836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980542.917, "dur": 173.981,
    "args": {
      "External id": 2488837,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980619.670, "dur": 6.868,
    "args": {
      "External id": 2488838,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980628.916, "dur": 4.239,
    "args": {
      "External id": 2488839,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980754.277, "dur": 25.659,
    "args": {
      "External id": 2488840,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980790.687, "dur": 13.622,
    "args": {
      "External id": 2488841,"kernel_hash": "caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/aa/caaxfmlp366sipjsiqrbwa7bn4zulwwbbk2av7b7vvjxoyepdkkh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980816.301, "dur": 38.991,
    "args": {
      "External id": 2488842,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980862.343, "dur": 37.543,
    "args": {
      "External id": 2488843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980905.721, "dur": 21.835,
    "args": {
      "External id": 2488844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980934.237, "dur": 29.113,
    "args": {
      "External id": 2488845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657980968.748, "dur": 56.309,
    "args": {
      "External id": 2488846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981036.417, "dur": 34.134,
    "args": {
      "External id": 2488847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981088.244, "dur": 23.730,
    "args": {
      "External id": 2488848,"kernel_hash": "cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldidzj4zx5p32cpjzdu4iw32vq5mroxtnzzjnkygl7i3gxyrjx4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981128.785, "dur": 25.197,
    "args": {
      "External id": 2488849,"kernel_hash": "cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/bd/cbd2aaszixjoxe5zl2hakyfbnpdttdl6psxd5acx4ptx7o6rknwz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981167.782, "dur": 16.553,
    "args": {
      "External id": 2488850,"kernel_hash": "cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/mm/cmmooewemsuhwg5it5lnepehjxxoc2bqz5dtdtexrhyn7fn4f6l2.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981197.057, "dur": 13.762,
    "args": {
      "External id": 2488851,"kernel_hash": "ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cj/ccjfnrp6f5yimftxut2uazyiv6kcmaxyqhpulc32bp3yvqhit56o.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981222.466, "dur": 18.168,
    "args": {
      "External id": 2488852,"kernel_hash": "c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5a5x6ehhfbgwbprspks6opettr56mogdbcmfzd5gpopd6oqjars.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981316.233, "dur": 16.616,
    "args": {
      "External id": 2488853,"Record function id": 0, "Ev Idx": 5652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981319.509, "dur": 12.327,
    "args": {
      "External id": 2488854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981323.318, "dur": 7.562,
    "args": {
      "External id": 2488855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981324.836, "dur": 5.905,
    "args": {
      "External id": 2488856,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981336.728, "dur": 4.901,
    "args": {
      "External id": 2488857,"Record function id": 0, "Ev Idx": 5656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981338.162, "dur": 3.027,
    "args": {
      "External id": 2488858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981339.076, "dur": 1.645,
    "args": {
      "External id": 2488859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981339.733, "dur": 0.882,
    "args": {
      "External id": 2488860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981345.148, "dur": 4.581,
    "args": {
      "External id": 2488861,"Record function id": 0, "Ev Idx": 5660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981346.475, "dur": 2.839,
    "args": {
      "External id": 2488862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981347.450, "dur": 1.431,
    "args": {
      "External id": 2488863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981348.000, "dur": 0.805,
    "args": {
      "External id": 2488864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981352.900, "dur": 4.534,
    "args": {
      "External id": 2488865,"Record function id": 0, "Ev Idx": 5664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981354.800, "dur": 2.241,
    "args": {
      "External id": 2488866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981355.299, "dur": 1.312,
    "args": {
      "External id": 2488867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981355.652, "dur": 0.858,
    "args": {
      "External id": 2488868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981360.641, "dur": 3.894,
    "args": {
      "External id": 2488869,"Record function id": 0, "Ev Idx": 5668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981361.970, "dur": 2.149,
    "args": {
      "External id": 2488870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981362.713, "dur": 0.986,
    "args": {
      "External id": 2488871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981362.991, "dur": 0.639,
    "args": {
      "External id": 2488872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981367.726, "dur": 4.125,
    "args": {
      "External id": 2488873,"Record function id": 0, "Ev Idx": 5672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981369.062, "dur": 2.371,
    "args": {
      "External id": 2488874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981369.606, "dur": 1.332,
    "args": {
      "External id": 2488875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981370.109, "dur": 0.731,
    "args": {
      "External id": 2488876,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981375.178, "dur": 4.088,
    "args": {
      "External id": 2488877,"Record function id": 0, "Ev Idx": 5676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981376.557, "dur": 2.317,
    "args": {
      "External id": 2488878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981377.080, "dur": 1.334,
    "args": {
      "External id": 2488879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981377.739, "dur": 0.578,
    "args": {
      "External id": 2488880,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981382.406, "dur": 6.286,
    "args": {
      "External id": 2488881,"Record function id": 0, "Ev Idx": 5680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981383.940, "dur": 4.349,
    "args": {
      "External id": 2488882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981384.473, "dur": 3.387,
    "args": {
      "External id": 2488883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981385.282, "dur": 2.505,
    "args": {
      "External id": 2488884,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981391.839, "dur": 4.701,
    "args": {
      "External id": 2488885,"Record function id": 0, "Ev Idx": 5684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981393.262, "dur": 2.870,
    "args": {
      "External id": 2488886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981393.830, "dur": 1.872,
    "args": {
      "External id": 2488887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981394.432, "dur": 1.185,
    "args": {
      "External id": 2488888,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981400.269, "dur": 36534.935,
    "args": {
      "External id": 2488889,"Record function id": 0, "Sequence number": 24740421, "Fwd thread id": 1, "Ev Idx": 5688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981401.932, "dur": 36524.224,
    "args": {
      "External id": 2488890,"Sequence number": 24740421, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5689
    }
  },
  {
    "ph": "f", "id": 227, "pid": 1336760, "tid": 1381209, "ts": 1514657981401.932,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981430.363, "dur": 39.335,
    "args": {
      "External id": 2488891,"Record function id": 0, "Ev Idx": 5690
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981476.700, "dur": 107.386,
    "args": {
      "External id": 2488892,"Record function id": 0, "Ev Idx": 5691
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.0)", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981599.716, "dur": 36319.084,
    "args": {
      "External id": 2488893,"Record function id": 0, "Ev Idx": 5692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981695.903, "dur": 10.243,
    "args": {
      "External id": 2488894,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981716.647, "dur": 5.456,
    "args": {
      "External id": 2488895,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981735.366, "dur": 35400.753,
    "args": {
      "External id": 2488896,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981749.039, "dur": 35377.075,
    "args": {
      "External id": 2488897,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981803.564, "dur": 13.574,
    "args": {
      "External id": 2488898,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981824.360, "dur": 35264.628,
    "args": {
      "External id": 2488899,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981826.685, "dur": 35261.543,
    "args": {
      "External id": 2488900,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981830.582, "dur": 5.223,
    "args": {
      "External id": 2488901,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514657981837.531, "dur": 35246.978,
    "args": {
      "External id": 2488902,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017225.927, "dur": 9.615,
    "args": {
      "External id": 2488903,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017228.961, "dur": 6.263,
    "args": {
      "External id": 2488904,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017265.195, "dur": 366.226,
    "args": {
      "External id": 2488905,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017293.597, "dur": 332.790,
    "args": {
      "External id": 2488906,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5705, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017305.770, "dur": 314.913,
    "args": {
      "External id": 2488907,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017651.926, "dur": 2.283,
    "args": {
      "External id": 2488908,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5707, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017717.110, "dur": 6.530,
    "args": {
      "External id": 2488909,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017767.675, "dur": 1.523,
    "args": {
      "External id": 2488910,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017784.764, "dur": 3.039,
    "args": {
      "External id": 2488911,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017799.200, "dur": 1.158,
    "args": {
      "External id": 2488912,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017811.916, "dur": 0.841,
    "args": {
      "External id": 2488913,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017822.285, "dur": 1.030,
    "args": {
      "External id": 2488914,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017833.843, "dur": 3.178,
    "args": {
      "External id": 2488915,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017847.723, "dur": 0.926,
    "args": {
      "External id": 2488916,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017858.609, "dur": 0.835,
    "args": {
      "External id": 2488917,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017949.739, "dur": 311.161,
    "args": {
      "External id": 2488918,"Record function id": 0, "Sequence number": 24740420, "Fwd thread id": 1, "Ev Idx": 5717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336760, "tid": 1381209,
    "ts": 1514658017952.614, "dur": 296.195,
    "args": {
      "External id": 2488919,"Sequence number": 24740420, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5718
    }
  },
  {
    "ph": "f", "id": 228, "pid": 1336760, "tid": 1381209, "ts": 1514658017952.614,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_0", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018108.838, "dur": 46.119,
    "args": {
      "External id": 2488920,"kernel_hash": "cgh7npvq6fhcxx3reazrdhuyliq3n5n76s36t3lcymwjpvltgnle", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh7npvq6fhcxx3reazrdhuyliq3n5n76s36t3lcymwjpvltgnle.py", "kernel_backend": "triton", "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 5719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_1", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018170.204, "dur": 29.983,
    "args": {
      "External id": 2488921,"kernel_hash": "cnysn4mnhgerjcric5y2lu3awivs6z2tsvqalsf43c23fcqe7gd4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ny/cnysn4mnhgerjcric5y2lu3awivs6z2tsvqalsf43c23fcqe7gd4.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "float", "Scalar"], "Input Strides": [[4096, 1], [8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096], [16, 4096, 2048], [32000, 2048], []], "Ev Idx": 5720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_2", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018215.113, "dur": 18.454,
    "args": {
      "External id": 2488922,"kernel_hash": "catnay4z3ivljnzu3qgi3oklbggqihu5hajduwtrkgcudhz6vnwm", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/at/catnay4z3ivljnzu3qgi3oklbggqihu5hajduwtrkgcudhz6vnwm.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 5721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018273.261, "dur": 14.782,
    "args": {
      "External id": 2488923,"Record function id": 0, "Ev Idx": 5722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018276.864, "dur": 10.325,
    "args": {
      "External id": 2488924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018280.513, "dur": 5.813,
    "args": {
      "External id": 2488925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018282.180, "dur": 4.010,
    "args": {
      "External id": 2488926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5725
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_post_backward_callback", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018310.024, "dur": 8639.801,
    "args": {
      "External id": 2488927,"Record function id": 0, "Ev Idx": 5726
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018332.790, "dur": 39.318,
    "args": {
      "External id": 2488928,"Record function id": 0, "Ev Idx": 5727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018378.297, "dur": 296.493,
    "args": {
      "External id": 2488929,"Record function id": 0, "Ev Idx": 5728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018682.890, "dur": 7994.450,
    "args": {
      "External id": 2488930,"Record function id": 0, "Ev Idx": 5729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018807.503, "dur": 6.971,
    "args": {
      "External id": 2488931,"Record function id": 0, "Concrete Inputs": ["[336611328]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018825.454, "dur": 5.372,
    "args": {
      "External id": 2488932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018854.776, "dur": 6497.019,
    "args": {
      "External id": 2488933,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336760, "tid": 1381209,
    "ts": 1514658018873.316, "dur": 6464.808,
    "args": {
      "External id": 2488934,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658019201.622, "dur": 22.968,
    "args": {
      "External id": 2488935,"Record function id": 0, "Concrete Inputs": ["[82421]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1381209,
    "ts": 1514658019297.296, "dur": 5991.963,
    "args": {
      "External id": 2488936,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], [], []], "Ev Idx": 5735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1381209,
    "ts": 1514658019300.460, "dur": 5987.704,
    "args": {
      "External id": 2488937,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], []], "Ev Idx": 5736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658019305.759, "dur": 15.525,
    "args": {
      "External id": 2488938,"Record function id": 0, "Concrete Inputs": ["[82421]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1381209,
    "ts": 1514658019323.494, "dur": 5959.186,
    "args": {
      "External id": 2488939,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[82421], [82421], []], "Ev Idx": 5738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658025491.206, "dur": 11.176,
    "args": {
      "External id": 2488940,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[336611328], [], [], [], [], []], "Ev Idx": 5739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1381209,
    "ts": 1514658025494.561, "dur": 7.461,
    "args": {
      "External id": 2488941,"Record function id": 0, "Concrete Inputs": ["[42076416]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336760, "tid": 1381209,
    "ts": 1514658025551.531, "dur": 461.608,
    "args": {
      "External id": 2488942,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[42076416], [336611328], [], [], [], []], "Ev Idx": 5741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514658025577.358, "dur": 399.041,
    "args": {
      "External id": 2488943,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 42076416, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[336611328], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5742, "In msg nelems": 336611328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336760, "tid": 1381209,
    "ts": 1514658025594.729, "dur": 375.737,
    "args": {
      "External id": 2488944,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[336611328]], "Ev Idx": 5743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026033.693, "dur": 2.883,
    "args": {
      "External id": 2488945,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5744, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026097.594, "dur": 7.402,
    "args": {
      "External id": 2488946,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026146.207, "dur": 1.059,
    "args": {
      "External id": 2488947,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026162.380, "dur": 1.313,
    "args": {
      "External id": 2488948,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8192256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026176.602, "dur": 2.774,
    "args": {
      "External id": 2488949,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8716544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026190.858, "dur": 1.460,
    "args": {
      "External id": 2488950,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9240832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026202.267, "dur": 1.194,
    "args": {
      "External id": 2488951,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9765120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026215.656, "dur": 1.338,
    "args": {
      "External id": 2488952,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026228.120, "dur": 3.367,
    "args": {
      "External id": 2488953,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "10289664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026242.846, "dur": 1.487,
    "args": {
      "External id": 2488954,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "11731456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026254.661, "dur": 1.436,
    "args": {
      "External id": 2488955,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "13173248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026266.878, "dur": 1.735,
    "args": {
      "External id": 2488956,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026278.631, "dur": 2.614,
    "args": {
      "External id": 2488957,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "14615296"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026292.053, "dur": 1.114,
    "args": {
      "External id": 2488958,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15139584"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026303.056, "dur": 1.630,
    "args": {
      "External id": 2488959,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15663872"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026314.617, "dur": 1.171,
    "args": {
      "External id": 2488960,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "16188160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026325.369, "dur": 3.120,
    "args": {
      "External id": 2488961,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026338.068, "dur": 1.516,
    "args": {
      "External id": 2488962,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "16712704"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026348.923, "dur": 1.210,
    "args": {
      "External id": 2488963,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "18154496"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026360.298, "dur": 1.391,
    "args": {
      "External id": 2488964,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "19596288"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026371.594, "dur": 3.098,
    "args": {
      "External id": 2488965,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026385.483, "dur": 1.300,
    "args": {
      "External id": 2488966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21038336"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026396.238, "dur": 1.499,
    "args": {
      "External id": 2488967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21562624"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026407.634, "dur": 1.233,
    "args": {
      "External id": 2488968,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22086912"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026418.471, "dur": 3.087,
    "args": {
      "External id": 2488969,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22611200"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026431.224, "dur": 1.310,
    "args": {
      "External id": 2488970,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026443.569, "dur": 1.447,
    "args": {
      "External id": 2488971,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "23135744"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026454.808, "dur": 0.990,
    "args": {
      "External id": 2488972,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "24577536"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026465.932, "dur": 3.129,
    "args": {
      "External id": 2488973,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "26019328"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026479.197, "dur": 1.332,
    "args": {
      "External id": 2488974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026490.780, "dur": 1.008,
    "args": {
      "External id": 2488975,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27461376"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026501.325, "dur": 1.900,
    "args": {
      "External id": 2488976,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27985664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026512.036, "dur": 3.207,
    "args": {
      "External id": 2488977,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "28509952"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026525.932, "dur": 1.397,
    "args": {
      "External id": 2488978,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "29034240"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026556.767, "dur": 1.808,
    "args": {
      "External id": 2488979,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026568.581, "dur": 1.502,
    "args": {
      "External id": 2488980,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "29558784"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026580.716, "dur": 2.565,
    "args": {
      "External id": 2488981,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "31000576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026593.642, "dur": 1.469,
    "args": {
      "External id": 2488982,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "32442368"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026605.461, "dur": 1.570,
    "args": {
      "External id": 2488983,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1381209,
    "ts": 1514658026616.831, "dur": 1.209,
    "args": {
      "External id": 2488984,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "33884416"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5783
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "ProfilerStep#102911", "pid": 1336760, "tid": 1336760,
    "ts": 1514655938384.320, "dur": 2107699.071,
    "args": {
      "External id": 2478081,"Record function id": 0, "Ev Idx": 5784
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.zero_grad#AdamW.zero_grad", "pid": 1336760, "tid": 1336760,
    "ts": 1514655938418.730, "dur": 764.178,
    "args": {
      "External id": 2478082,"Record function id": 0, "Ev Idx": 5785
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "enumerate(DataLoader)#_StatefulMultiProcessingDataLoaderIter.__next__", "pid": 1336760, "tid": 1336760,
    "ts": 1514655939229.720, "dur": 2134.088,
    "args": {
      "External id": 2478083,"Record function id": 0, "Ev Idx": 5786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655940283.481, "dur": 7.772,
    "args": {
      "External id": 2478084,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655940313.423, "dur": 6.688,
    "args": {
      "External id": 2478085,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655940785.851, "dur": 2.534,
    "args": {
      "External id": 2478086,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655940796.168, "dur": 2.828,
    "args": {
      "External id": 2478087,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655941256.576, "dur": 1.929,
    "args": {
      "External id": 2478088,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655941264.744, "dur": 1.627,
    "args": {
      "External id": 2478089,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942239.390, "dur": 14.153,
    "args": {
      "External id": 2478090,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942246.996, "dur": 2.724,
    "args": {
      "External id": 2478091,"Record function id": 0, "Concrete Inputs": ["", "[16, 8192]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942255.310, "dur": 4.386,
    "args": {
      "External id": 2478092,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942257.606, "dur": 0.999,
    "args": {
      "External id": 2478093,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942288.335, "dur": 569.259,
    "args": {
      "External id": 2478094,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], [], []], "Ev Idx": 5797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942295.444, "dur": 561.301,
    "args": {
      "External id": 2478095,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942303.688, "dur": 10.654,
    "args": {
      "External id": 2478096,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942316.411, "dur": 538.461,
    "args": {
      "External id": 2478097,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942325.968, "dur": 0.614,
    "args": {
      "External id": 2478098,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942330.093, "dur": 10.492,
    "args": {
      "External id": 2478099,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[8192, 1], [4096, 1]], "Input Dims": [[16, 4096], [16, 4096]], "Ev Idx": 5802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942336.127, "dur": 4.099,
    "args": {
      "External id": 2478100,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], []], "Input Dims": [[16, 4096], [], []], "Ev Idx": 5803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942339.250, "dur": 0.565,
    "args": {
      "External id": 2478101,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942343.066, "dur": 247.894,
    "args": {
      "External id": 2478102,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942345.707, "dur": 244.911,
    "args": {
      "External id": 2478103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942347.986, "dur": 15.349,
    "args": {
      "External id": 2478104,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 5807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942350.517, "dur": 12.280,
    "args": {
      "External id": 2478105,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942364.465, "dur": 224.977,
    "args": {
      "External id": 2478106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942593.958, "dur": 257.583,
    "args": {
      "External id": 2478107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942876.034, "dur": 671.269,
    "args": {
      "External id": 2478108,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 5811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942877.929, "dur": 668.351,
    "args": {
      "External id": 2478109,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], []], "Ev Idx": 5812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942885.736, "dur": 8.907,
    "args": {
      "External id": 2478110,"Record function id": 0, "Concrete Inputs": ["[16, 8192]", "[8192, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655942895.601, "dur": 629.660,
    "args": {
      "External id": 2478111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[8192, 1], [8192, 1], []], "Input Dims": [[16, 8192], [16, 8192], []], "Ev Idx": 5814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943579.067, "dur": 59.874,
    "args": {
      "External id": 2478112,"Record function id": 0, "Concrete Inputs": ["0", "4096", "", "", "", "False"], "Input type": ["Scalar", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943585.709, "dur": 6.530,
    "args": {
      "External id": 2478113,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943595.127, "dur": 43.480,
    "args": {
      "External id": 2478114,"Record function id": 0, "Concrete Inputs": ["0", "4096", "1", ""], "Input type": ["Scalar", "Scalar", "Scalar", "long int"], "Input Strides": [[], [], [], [1]], "Input Dims": [[], [], [], [0]], "Ev Idx": 5817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943600.365, "dur": 7.191,
    "args": {
      "External id": 2478115,"Record function id": 0, "Concrete Inputs": ["", "[4096]", ""], "Input type": ["long int", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 5818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::repeat", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943652.349, "dur": 75.379,
    "args": {
      "External id": 2478116,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4096], []], "Ev Idx": 5819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943656.791, "dur": 8.174,
    "args": {
      "External id": 2478117,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[4096], [], []], "Ev Idx": 5820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943662.607, "dur": 2.071,
    "args": {
      "External id": 2478118,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "[4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[4096], [], [], []], "Ev Idx": 5821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943665.909, "dur": 5.366,
    "args": {
      "External id": 2478119,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943673.702, "dur": 3.553,
    "args": {
      "External id": 2478120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[4096, 1]], "Input Dims": [[16, 4096]], "Ev Idx": 5823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943681.691, "dur": 4.935,
    "args": {
      "External id": 2478121,"Record function id": 0, "Concrete Inputs": ["", "0", "1", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943685.720, "dur": 0.773,
    "args": {
      "External id": 2478122,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 1]", "[4096, 1, 4096]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943687.619, "dur": 5.180,
    "args": {
      "External id": 2478123,"Record function id": 0, "Concrete Inputs": ["", "1", "4096", "4096"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943691.778, "dur": 0.934,
    "args": {
      "External id": 2478124,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[4096, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943694.835, "dur": 4.454,
    "args": {
      "External id": 2478125,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[4096, 1], [4096, 4096, 4096, 1]], "Input Dims": [[1, 4096], [16, 1, 1, 4096]], "Ev Idx": 5828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943696.155, "dur": 3.021,
    "args": {
      "External id": 2478126,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[4096, 1], [], []], "Input Dims": [[1, 4096], [], []], "Ev Idx": 5829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943697.906, "dur": 0.934,
    "args": {
      "External id": 2478127,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[0, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[1, 4096], [], [], []], "Ev Idx": 5830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943700.127, "dur": 26.879,
    "args": {
      "External id": 2478128,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 4096, 4096, 1], [0, 4096, 4096, 1], []], "Input Dims": [[16, 1, 1, 4096], [16, 1, 1, 4096], []], "Ev Idx": 5831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943735.827, "dur": 33.081,
    "args": {
      "External id": 2478129,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943737.181, "dur": 31.555,
    "args": {
      "External id": 2478130,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "", "", "", "False", ""], "Input type": ["long int", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943744.773, "dur": 3.444,
    "args": {
      "External id": 2478131,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "3", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943748.971, "dur": 19.388,
    "args": {
      "External id": 2478132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5835
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_pre_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943887.949, "dur": 191.086,
    "args": {
      "External id": 2478133,"Record function id": 0, "Ev Idx": 5836
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::inputs_to_device", "pid": 1336760, "tid": 1336760,
    "ts": 1514655943971.992, "dur": 95.924,
    "args": {
      "External id": 2478134,"Record function id": 0, "Ev Idx": 5837
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944086.900, "dur": 45.208,
    "args": {
      "External id": 2478135,"Record function id": 0, "Ev Idx": 5838
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944141.024, "dur": 8203.630,
    "args": {
      "External id": 2478136,"Record function id": 0, "Ev Idx": 5839
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944152.188, "dur": 1357.788,
    "args": {
      "External id": 2478137,"Record function id": 0, "Ev Idx": 5840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944280.783, "dur": 8.200,
    "args": {
      "External id": 2478138,"Record function id": 0, "Concrete Inputs": ["[42076416]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944307.371, "dur": 141.116,
    "args": {
      "External id": 2478139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944312.658, "dur": 1.691,
    "args": {
      "External id": 2478140,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944320.668, "dur": 0.633,
    "args": {
      "External id": 2478141,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944322.488, "dur": 0.638,
    "args": {
      "External id": 2478142,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8192256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944324.514, "dur": 2.180,
    "args": {
      "External id": 2478143,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8716544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944331.761, "dur": 0.397,
    "args": {
      "External id": 2478144,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9240832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944333.019, "dur": 0.270,
    "args": {
      "External id": 2478145,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9765120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944334.163, "dur": 2.411,
    "args": {
      "External id": 2478146,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944339.622, "dur": 0.599,
    "args": {
      "External id": 2478147,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "10289664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944340.897, "dur": 0.492,
    "args": {
      "External id": 2478148,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11731456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944347.402, "dur": 0.347,
    "args": {
      "External id": 2478149,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "13173248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944348.178, "dur": 0.263,
    "args": {
      "External id": 2478150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944349.254, "dur": 2.348,
    "args": {
      "External id": 2478151,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "14615296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944356.654, "dur": 0.591,
    "args": {
      "External id": 2478152,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15139584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944357.873, "dur": 0.173,
    "args": {
      "External id": 2478153,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15663872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944358.475, "dur": 2.201,
    "args": {
      "External id": 2478154,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "16188160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944364.180, "dur": 0.266,
    "args": {
      "External id": 2478155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944364.926, "dur": 0.210,
    "args": {
      "External id": 2478156,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "16712704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944370.361, "dur": 0.143,
    "args": {
      "External id": 2478157,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "18154496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944371.157, "dur": 0.367,
    "args": {
      "External id": 2478158,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "19596288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944372.545, "dur": 2.317,
    "args": {
      "External id": 2478159,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944379.688, "dur": 0.227,
    "args": {
      "External id": 2478160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21038336"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944380.684, "dur": 0.282,
    "args": {
      "External id": 2478161,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21562624"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944381.699, "dur": 2.422,
    "args": {
      "External id": 2478162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22086912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944387.559, "dur": 0.337,
    "args": {
      "External id": 2478163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22611200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944388.554, "dur": 0.348,
    "args": {
      "External id": 2478164,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944394.958, "dur": 0.380,
    "args": {
      "External id": 2478165,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "23135744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944395.977, "dur": 0.374,
    "args": {
      "External id": 2478166,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24577536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944396.993, "dur": 2.242,
    "args": {
      "External id": 2478167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "26019328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944402.440, "dur": 0.183,
    "args": {
      "External id": 2478168,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944403.255, "dur": 0.299,
    "args": {
      "External id": 2478169,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27461376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944404.179, "dur": 2.639,
    "args": {
      "External id": 2478170,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27985664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944409.868, "dur": 0.196,
    "args": {
      "External id": 2478171,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "28509952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944410.851, "dur": 0.620,
    "args": {
      "External id": 2478172,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "29034240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944416.729, "dur": 0.177,
    "args": {
      "External id": 2478173,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944417.335, "dur": 0.511,
    "args": {
      "External id": 2478174,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29558784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944418.737, "dur": 2.361,
    "args": {
      "External id": 2478175,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "31000576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944424.897, "dur": 0.594,
    "args": {
      "External id": 2478176,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "32442368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944426.136, "dur": 0.153,
    "args": {
      "External id": 2478177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944427.065, "dur": 2.256,
    "args": {
      "External id": 2478178,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "33884416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944474.595, "dur": 98.446,
    "args": {
      "External id": 2478179,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944656.205, "dur": 281.297,
    "args": {
      "External id": 2478180,"Record function id": 0, "Concrete Inputs": ["", "", "42076416", "8", "7", "15", ""], "Input type": ["TensorList", "", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 5883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944674.082, "dur": 5.440,
    "args": {
      "External id": 2478181,"Record function id": 0, "Concrete Inputs": ["[336611328]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944686.355, "dur": 13.459,
    "args": {
      "External id": 2478182,"Record function id": 0, "Concrete Inputs": ["", "0", "294534912", "42076416"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944690.868, "dur": 8.475,
    "args": {
      "External id": 2478183,"Record function id": 0, "Concrete Inputs": ["", "0", "294534912", "336611328", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[336611328], [], [], [], []], "Ev Idx": 5886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944695.896, "dur": 0.749,
    "args": {
      "External id": 2478184,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "[1]", "294534912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944707.874, "dur": 126.483,
    "args": {
      "External id": 2478185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944709.727, "dur": 0.589,
    "args": {
      "External id": 2478186,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "294534912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944711.676, "dur": 0.597,
    "args": {
      "External id": 2478187,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "302726912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944715.277, "dur": 2.541,
    "args": {
      "External id": 2478188,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "302727168"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944718.528, "dur": 0.920,
    "args": {
      "External id": 2478189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "303251456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944722.502, "dur": 0.180,
    "args": {
      "External id": 2478190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "303775744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944723.520, "dur": 0.501,
    "args": {
      "External id": 2478191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "304300032"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944725.013, "dur": 0.378,
    "args": {
      "External id": 2478192,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "304824320"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944727.867, "dur": 0.339,
    "args": {
      "External id": 2478193,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "304824576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944728.875, "dur": 0.903,
    "args": {
      "External id": 2478194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "306266368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944730.778, "dur": 0.480,
    "args": {
      "External id": 2478195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "307708160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944734.119, "dur": 1.438,
    "args": {
      "External id": 2478196,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "309149952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944736.558, "dur": 0.529,
    "args": {
      "External id": 2478197,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "309150208"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944739.823, "dur": 2.261,
    "args": {
      "External id": 2478198,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "309674496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944742.778, "dur": 0.338,
    "args": {
      "External id": 2478199,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "310198784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944743.990, "dur": 0.800,
    "args": {
      "External id": 2478200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "310723072"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944750.187, "dur": 0.342,
    "args": {
      "External id": 2478201,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "311247360"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944751.155, "dur": 0.266,
    "args": {
      "External id": 2478202,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "311247616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944752.051, "dur": 0.345,
    "args": {
      "External id": 2478203,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "312689408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944757.743, "dur": 1.598,
    "args": {
      "External id": 2478204,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "314131200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944760.154, "dur": 0.173,
    "args": {
      "External id": 2478205,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "315572992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944763.211, "dur": 2.141,
    "args": {
      "External id": 2478206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "315573248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944766.233, "dur": 0.214,
    "args": {
      "External id": 2478207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "316097536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944767.144, "dur": 0.309,
    "args": {
      "External id": 2478208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "316621824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944772.655, "dur": 0.374,
    "args": {
      "External id": 2478209,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "317146112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944773.693, "dur": 0.322,
    "args": {
      "External id": 2478210,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "317670400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944774.640, "dur": 0.274,
    "args": {
      "External id": 2478211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "317670656"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944779.763, "dur": 5.063,
    "args": {
      "External id": 2478212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "319112448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944785.511, "dur": 0.592,
    "args": {
      "External id": 2478213,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "320554240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944789.047, "dur": 2.585,
    "args": {
      "External id": 2478214,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "321996032"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944792.269, "dur": 0.606,
    "args": {
      "External id": 2478215,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "321996288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944793.649, "dur": 0.552,
    "args": {
      "External id": 2478216,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "322520576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944799.314, "dur": 0.311,
    "args": {
      "External id": 2478217,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "323044864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944800.078, "dur": 0.363,
    "args": {
      "External id": 2478218,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "323569152"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944801.132, "dur": 0.173,
    "args": {
      "External id": 2478219,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "324093440"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944807.182, "dur": 2.293,
    "args": {
      "External id": 2478220,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "324093696"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944810.181, "dur": 0.330,
    "args": {
      "External id": 2478221,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "325535488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944813.058, "dur": 2.320,
    "args": {
      "External id": 2478222,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "326977280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944816.431, "dur": 0.370,
    "args": {
      "External id": 2478223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "328419072"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944817.440, "dur": 0.319,
    "args": {
      "External id": 2478224,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "328419328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655944861.099, "dur": 59.147,
    "args": {
      "External id": 2478225,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945032.557, "dur": 371.076,
    "args": {
      "External id": 2478226,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[336611328], [42076416], [], [], []], "Ev Idx": 5929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945069.366, "dur": 329.298,
    "args": {
      "External id": 2478227,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 336611328, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[42076416], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5930, "In msg nelems": 42076416
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945081.072, "dur": 312.112,
    "args": {
      "External id": 2478228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[42076416]], "Ev Idx": 5931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945426.602, "dur": 2.261,
    "args": {
      "External id": 2478229,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5932, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945525.238, "dur": 6602.767,
    "args": {
      "External id": 2478230,"Record function id": 0, "Ev Idx": 5933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945777.410, "dur": 7.281,
    "args": {
      "External id": 2478231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945789.013, "dur": 1.289,
    "args": {
      "External id": 2478232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945792.367, "dur": 1.193,
    "args": {
      "External id": 2478233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945795.414, "dur": 2.488,
    "args": {
      "External id": 2478234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945799.442, "dur": 1.227,
    "args": {
      "External id": 2478235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945804.443, "dur": 0.858,
    "args": {
      "External id": 2478236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945806.870, "dur": 1.192,
    "args": {
      "External id": 2478237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945809.312, "dur": 2.290,
    "args": {
      "External id": 2478238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945815.855, "dur": 1.222,
    "args": {
      "External id": 2478239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945820.773, "dur": 1.030,
    "args": {
      "External id": 2478240,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945823.575, "dur": 1.028,
    "args": {
      "External id": 2478241,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945826.037, "dur": 2.220,
    "args": {
      "External id": 2478242,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945829.479, "dur": 0.931,
    "args": {
      "External id": 2478243,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945833.751, "dur": 1.036,
    "args": {
      "External id": 2478244,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945836.265, "dur": 0.865,
    "args": {
      "External id": 2478245,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945838.307, "dur": 2.245,
    "args": {
      "External id": 2478246,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945844.140, "dur": 1.027,
    "args": {
      "External id": 2478247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945848.781, "dur": 1.018,
    "args": {
      "External id": 2478248,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945851.740, "dur": 0.866,
    "args": {
      "External id": 2478249,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945853.875, "dur": 2.075,
    "args": {
      "External id": 2478250,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945857.425, "dur": 1.196,
    "args": {
      "External id": 2478251,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945862.116, "dur": 0.860,
    "args": {
      "External id": 2478252,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945864.434, "dur": 0.845,
    "args": {
      "External id": 2478253,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945866.644, "dur": 1.975,
    "args": {
      "External id": 2478254,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945885.805, "dur": 1.016,
    "args": {
      "External id": 2478255,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945889.775, "dur": 0.903,
    "args": {
      "External id": 2478256,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945892.425, "dur": 1.013,
    "args": {
      "External id": 2478257,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945894.704, "dur": 2.520,
    "args": {
      "External id": 2478258,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945898.587, "dur": 0.981,
    "args": {
      "External id": 2478259,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945902.804, "dur": 0.969,
    "args": {
      "External id": 2478260,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945905.104, "dur": 1.011,
    "args": {
      "External id": 2478261,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945907.281, "dur": 2.145,
    "args": {
      "External id": 2478262,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945912.909, "dur": 0.943,
    "args": {
      "External id": 2478263,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945917.639, "dur": 0.893,
    "args": {
      "External id": 2478264,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945923.548, "dur": 0.714,
    "args": {
      "External id": 2478265,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945925.597, "dur": 2.026,
    "args": {
      "External id": 2478266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945929.264, "dur": 0.884,
    "args": {
      "External id": 2478267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945933.094, "dur": 0.821,
    "args": {
      "External id": 2478268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945935.210, "dur": 0.998,
    "args": {
      "External id": 2478269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945937.660, "dur": 2.009,
    "args": {
      "External id": 2478270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655945968.941, "dur": 6106.503,
    "args": {
      "External id": 2478271,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946031.691, "dur": 6035.102,
    "args": {
      "External id": 2478272,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946057.694, "dur": 6.164,
    "args": {
      "External id": 2478273,"Record function id": 0, "Concrete Inputs": ["[4290]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946071.707, "dur": 5953.510,
    "args": {
      "External id": 2478274,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], [], []], "Ev Idx": 5977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946074.422, "dur": 5949.970,
    "args": {
      "External id": 2478275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], []], "Ev Idx": 5978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946081.733, "dur": 5.907,
    "args": {
      "External id": 2478276,"Record function id": 0, "Concrete Inputs": ["[4290]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655946089.442, "dur": 5931.995,
    "args": {
      "External id": 2478277,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[4290], [4290], []], "Ev Idx": 5980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952430.590, "dur": 45.636,
    "args": {
      "External id": 2478278,"Record function id": 0, "Ev Idx": 5981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 0/0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952477.644, "dur": 248.193,
    "args": {
      "External id": 2478279,"Record function id": 0, "Ev Idx": 5982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952522.974, "dur": 193.287,
    "args": {
      "External id": 2478280,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "long int"], "Input Strides": [[2048, 1], [4096, 1]], "Input Dims": [[32000, 2048], [16, 4096]], "Ev Idx": 5983
    }
  },
  {
    "ph": "s", "id": 228, "pid": 1336760, "tid": 1336760, "ts": 1514655952522.974,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952621.542, "dur": 56.090,
    "args": {
      "External id": 2478281,"kernel_hash": "cfktqnhujkvemjetr4frr6i6zlhasbqswa6l3ebbfx7l4gnf2ioq", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/fk/cfktqnhujkvemjetr4frr6i6zlhasbqswa6l3ebbfx7l4gnf2ioq.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[4096, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096], [32000, 2048], [16, 4096, 2048], []], "Ev Idx": 5984
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952790.633, "dur": 60.062,
    "args": {
      "External id": 2478282,"Record function id": 0, "Ev Idx": 5985
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.0)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952861.136, "dur": 7873.482,
    "args": {
      "External id": 2478283,"Record function id": 0, "Ev Idx": 5986
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952869.750, "dur": 862.377,
    "args": {
      "External id": 2478284,"Record function id": 0, "Ev Idx": 5987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952951.428, "dur": 10.005,
    "args": {
      "External id": 2478285,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655952974.937, "dur": 98.176,
    "args": {
      "External id": 2478286,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953036.735, "dur": 2.859,
    "args": {
      "External id": 2478287,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953042.348, "dur": 0.639,
    "args": {
      "External id": 2478288,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953043.921, "dur": 2.560,
    "args": {
      "External id": 2478289,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953049.907, "dur": 0.637,
    "args": {
      "External id": 2478290,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953051.338, "dur": 0.667,
    "args": {
      "External id": 2478291,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953057.393, "dur": 0.192,
    "args": {
      "External id": 2478292,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953058.513, "dur": 0.528,
    "args": {
      "External id": 2478293,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953059.706, "dur": 2.100,
    "args": {
      "External id": 2478294,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953065.034, "dur": 0.223,
    "args": {
      "External id": 2478295,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953085.459, "dur": 43.771,
    "args": {
      "External id": 2478296,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953162.218, "dur": 119.854,
    "args": {
      "External id": 2478297,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953174.379, "dur": 5.378,
    "args": {
      "External id": 2478298,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953185.393, "dur": 11.620,
    "args": {
      "External id": 2478299,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953189.710, "dur": 6.893,
    "args": {
      "External id": 2478300,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953194.834, "dur": 0.738,
    "args": {
      "External id": 2478301,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953203.443, "dur": 31.804,
    "args": {
      "External id": 2478302,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953205.468, "dur": 0.505,
    "args": {
      "External id": 2478303,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953209.592, "dur": 0.730,
    "args": {
      "External id": 2478304,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953213.613, "dur": 0.259,
    "args": {
      "External id": 2478305,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953216.499, "dur": 0.362,
    "args": {
      "External id": 2478306,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953217.633, "dur": 3.715,
    "args": {
      "External id": 2478307,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953222.052, "dur": 0.389,
    "args": {
      "External id": 2478308,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953225.467, "dur": 0.387,
    "args": {
      "External id": 2478309,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953228.941, "dur": 0.375,
    "args": {
      "External id": 2478310,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953229.950, "dur": 0.358,
    "args": {
      "External id": 2478311,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953248.513, "dur": 25.233,
    "args": {
      "External id": 2478312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953335.120, "dur": 305.764,
    "args": {
      "External id": 2478313,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953365.246, "dur": 270.782,
    "args": {
      "External id": 2478314,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6017, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953375.398, "dur": 254.838,
    "args": {
      "External id": 2478315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953663.623, "dur": 2.181,
    "args": {
      "External id": 2478316,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6019, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953753.211, "dur": 6742.802,
    "args": {
      "External id": 2478317,"Record function id": 0, "Ev Idx": 6020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953860.162, "dur": 6.335,
    "args": {
      "External id": 2478318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953870.083, "dur": 1.590,
    "args": {
      "External id": 2478319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953873.630, "dur": 1.263,
    "args": {
      "External id": 2478320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953878.601, "dur": 2.165,
    "args": {
      "External id": 2478321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953882.203, "dur": 1.014,
    "args": {
      "External id": 2478322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953884.581, "dur": 1.098,
    "args": {
      "External id": 2478323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953887.305, "dur": 1.278,
    "args": {
      "External id": 2478324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953893.919, "dur": 1.680,
    "args": {
      "External id": 2478325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953897.079, "dur": 1.078,
    "args": {
      "External id": 2478326,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953899.511, "dur": 0.990,
    "args": {
      "External id": 2478327,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953917.007, "dur": 6542.473,
    "args": {
      "External id": 2478328,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953934.216, "dur": 6518.935,
    "args": {
      "External id": 2478329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953955.716, "dur": 13.386,
    "args": {
      "External id": 2478330,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953974.801, "dur": 6442.751,
    "args": {
      "External id": 2478331,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655953977.449, "dur": 6439.538,
    "args": {
      "External id": 2478332,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655954019.089, "dur": 8.664,
    "args": {
      "External id": 2478333,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655954029.972, "dur": 6384.352,
    "args": {
      "External id": 2478334,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960678.698, "dur": 31.351,
    "args": {
      "External id": 2478335,"Sequence number": 24740421, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6038
    }
  },
  {
    "ph": "s", "id": 227, "pid": 1336760, "tid": 1336760, "ts": 1514655960678.698,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960694.924, "dur": 10.394,
    "args": {
      "External id": 2478336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960700.684, "dur": 4.306,
    "args": {
      "External id": 2478337,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960778.430, "dur": 100.482,
    "args": {
      "External id": 2478338,"Record function id": 0, "Ev Idx": 6041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960880.570, "dur": 1252.532,
    "args": {
      "External id": 2478339,"Record function id": 0, "Ev Idx": 6042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514655960923.754, "dur": 1193.455,
    "args": {
      "External id": 2478340,"Sequence number": 24740422, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6043
    }
  },
  {
    "ph": "s", "id": 226, "pid": 1336760, "tid": 1336760, "ts": 1514655960923.754,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961049.347, "dur": 49.910,
    "args": {
      "External id": 2478341,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961115.322, "dur": 110.154,
    "args": {
      "External id": 2478342,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961237.432, "dur": 39.153,
    "args": {
      "External id": 2478343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961284.483, "dur": 31.853,
    "args": {
      "External id": 2478344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961343.053, "dur": 30.095,
    "args": {
      "External id": 2478345,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961393.565, "dur": 17.786,
    "args": {
      "External id": 2478346,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961430.306, "dur": 169.289,
    "args": {
      "External id": 2478347,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961488.436, "dur": 10.394,
    "args": {
      "External id": 2478348,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961493.129, "dur": 4.737,
    "args": {
      "External id": 2478349,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961501.707, "dur": 4.007,
    "args": {
      "External id": 2478350,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961509.271, "dur": 1.505,
    "args": {
      "External id": 2478351,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961513.085, "dur": 4.028,
    "args": {
      "External id": 2478352,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961611.864, "dur": 53.667,
    "args": {
      "External id": 2478353,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961696.991, "dur": 33.349,
    "args": {
      "External id": 2478354,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961738.314, "dur": 44.170,
    "args": {
      "External id": 2478355,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961792.719, "dur": 35.924,
    "args": {
      "External id": 2478356,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961850.380, "dur": 30.958,
    "args": {
      "External id": 2478357,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961887.703, "dur": 35.244,
    "args": {
      "External id": 2478358,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655961953.280, "dur": 21.595,
    "args": {
      "External id": 2478359,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6062
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.0)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962201.637, "dur": 92.078,
    "args": {
      "External id": 2478360,"Record function id": 0, "Ev Idx": 6063
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962368.563, "dur": 45.467,
    "args": {
      "External id": 2478361,"Record function id": 0, "Ev Idx": 6064
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.1)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962423.581, "dur": 19145.885,
    "args": {
      "External id": 2478362,"Record function id": 0, "Ev Idx": 6065
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962435.477, "dur": 982.021,
    "args": {
      "External id": 2478363,"Record function id": 0, "Ev Idx": 6066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962520.063, "dur": 40.337,
    "args": {
      "External id": 2478364,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962575.414, "dur": 44.906,
    "args": {
      "External id": 2478365,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962583.614, "dur": 4.532,
    "args": {
      "External id": 2478366,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962589.799, "dur": 0.675,
    "args": {
      "External id": 2478367,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962593.866, "dur": 0.626,
    "args": {
      "External id": 2478368,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962597.468, "dur": 0.173,
    "args": {
      "External id": 2478369,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962598.154, "dur": 1.517,
    "args": {
      "External id": 2478370,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962602.333, "dur": 0.444,
    "args": {
      "External id": 2478371,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962605.346, "dur": 0.426,
    "args": {
      "External id": 2478372,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962606.578, "dur": 0.167,
    "args": {
      "External id": 2478373,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962610.632, "dur": 2.600,
    "args": {
      "External id": 2478374,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962631.750, "dur": 49.222,
    "args": {
      "External id": 2478375,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962714.200, "dur": 123.187,
    "args": {
      "External id": 2478376,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962727.752, "dur": 4.460,
    "args": {
      "External id": 2478377,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962736.999, "dur": 10.033,
    "args": {
      "External id": 2478378,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962741.552, "dur": 5.043,
    "args": {
      "External id": 2478379,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962744.460, "dur": 0.885,
    "args": {
      "External id": 2478380,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962753.082, "dur": 36.306,
    "args": {
      "External id": 2478381,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962756.793, "dur": 0.558,
    "args": {
      "External id": 2478382,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962758.418, "dur": 2.222,
    "args": {
      "External id": 2478383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962761.202, "dur": 2.609,
    "args": {
      "External id": 2478384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962767.124, "dur": 0.196,
    "args": {
      "External id": 2478385,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962768.185, "dur": 0.448,
    "args": {
      "External id": 2478386,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962771.949, "dur": 0.465,
    "args": {
      "External id": 2478387,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962775.379, "dur": 0.431,
    "args": {
      "External id": 2478388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962776.928, "dur": 0.510,
    "args": {
      "External id": 2478389,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962782.516, "dur": 0.594,
    "args": {
      "External id": 2478390,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962802.537, "dur": 25.383,
    "args": {
      "External id": 2478391,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962891.171, "dur": 430.737,
    "args": {
      "External id": 2478392,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962924.750, "dur": 391.648,
    "args": {
      "External id": 2478393,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6096, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514655962935.266, "dur": 374.643,
    "args": {
      "External id": 2478394,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963347.482, "dur": 2.421,
    "args": {
      "External id": 2478395,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6098, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963437.744, "dur": 17917.986,
    "args": {
      "External id": 2478396,"Record function id": 0, "Ev Idx": 6099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963578.202, "dur": 6.959,
    "args": {
      "External id": 2478397,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963588.950, "dur": 1.725,
    "args": {
      "External id": 2478398,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963592.577, "dur": 0.998,
    "args": {
      "External id": 2478399,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963595.368, "dur": 0.884,
    "args": {
      "External id": 2478400,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963597.920, "dur": 0.848,
    "args": {
      "External id": 2478401,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963602.328, "dur": 1.049,
    "args": {
      "External id": 2478402,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963607.041, "dur": 1.423,
    "args": {
      "External id": 2478403,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963609.956, "dur": 4.080,
    "args": {
      "External id": 2478404,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963615.280, "dur": 0.972,
    "args": {
      "External id": 2478405,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963619.542, "dur": 1.087,
    "args": {
      "External id": 2478406,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963639.136, "dur": 17679.849,
    "args": {
      "External id": 2478407,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963657.050, "dur": 17655.384,
    "args": {
      "External id": 2478408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963680.139, "dur": 13.084,
    "args": {
      "External id": 2478409,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963697.235, "dur": 17580.865,
    "args": {
      "External id": 2478410,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963699.888, "dur": 17577.641,
    "args": {
      "External id": 2478411,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963705.756, "dur": 4.688,
    "args": {
      "External id": 2478412,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655963712.200, "dur": 17562.569,
    "args": {
      "External id": 2478413,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981500.730, "dur": 43.124,
    "args": {
      "External id": 2478414,"Sequence number": 24740423, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6117
    }
  },
  {
    "ph": "s", "id": 225, "pid": 1336760, "tid": 1336760, "ts": 1514655981500.730,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981516.756, "dur": 21.052,
    "args": {
      "External id": 2478415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981521.649, "dur": 4.316,
    "args": {
      "External id": 2478416,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981614.752, "dur": 93.007,
    "args": {
      "External id": 2478417,"Record function id": 0, "Ev Idx": 6120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981709.331, "dur": 1091.004,
    "args": {
      "External id": 2478418,"Record function id": 0, "Ev Idx": 6121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981749.942, "dur": 1036.047,
    "args": {
      "External id": 2478419,"Sequence number": 24740424, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6122
    }
  },
  {
    "ph": "s", "id": 224, "pid": 1336760, "tid": 1336760, "ts": 1514655981749.942,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981830.160, "dur": 44.746,
    "args": {
      "External id": 2478420,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655981888.948, "dur": 130.841,
    "args": {
      "External id": 2478421,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982032.040, "dur": 44.810,
    "args": {
      "External id": 2478422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982082.816, "dur": 31.818,
    "args": {
      "External id": 2478423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982147.338, "dur": 27.520,
    "args": {
      "External id": 2478424,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982190.329, "dur": 18.601,
    "args": {
      "External id": 2478425,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982223.300, "dur": 132.920,
    "args": {
      "External id": 2478426,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982274.909, "dur": 11.723,
    "args": {
      "External id": 2478427,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982279.647, "dur": 6.231,
    "args": {
      "External id": 2478428,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982291.793, "dur": 4.582,
    "args": {
      "External id": 2478429,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982297.563, "dur": 3.140,
    "args": {
      "External id": 2478430,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982302.982, "dur": 2.579,
    "args": {
      "External id": 2478431,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982366.500, "dur": 45.044,
    "args": {
      "External id": 2478432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982441.252, "dur": 29.720,
    "args": {
      "External id": 2478433,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982479.272, "dur": 42.590,
    "args": {
      "External id": 2478434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982544.450, "dur": 39.468,
    "args": {
      "External id": 2478435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982611.870, "dur": 29.186,
    "args": {
      "External id": 2478436,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982646.796, "dur": 34.651,
    "args": {
      "External id": 2478437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982698.528, "dur": 21.686,
    "args": {
      "External id": 2478438,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6141
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.1)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655982863.483, "dur": 81.894,
    "args": {
      "External id": 2478439,"Record function id": 0, "Ev Idx": 6142
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983060.378, "dur": 49.323,
    "args": {
      "External id": 2478440,"Record function id": 0, "Ev Idx": 6143
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.2)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983119.691, "dur": 18032.410,
    "args": {
      "External id": 2478441,"Record function id": 0, "Ev Idx": 6144
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983131.335, "dur": 944.255,
    "args": {
      "External id": 2478442,"Record function id": 0, "Ev Idx": 6145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983215.342, "dur": 8.765,
    "args": {
      "External id": 2478443,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983239.365, "dur": 41.996,
    "args": {
      "External id": 2478444,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983244.667, "dur": 2.764,
    "args": {
      "External id": 2478445,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983251.491, "dur": 0.631,
    "args": {
      "External id": 2478446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983255.333, "dur": 0.412,
    "args": {
      "External id": 2478447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983256.625, "dur": 0.604,
    "args": {
      "External id": 2478448,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983260.704, "dur": 0.209,
    "args": {
      "External id": 2478449,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983264.458, "dur": 0.502,
    "args": {
      "External id": 2478450,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983265.880, "dur": 4.255,
    "args": {
      "External id": 2478451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983272.856, "dur": 0.381,
    "args": {
      "External id": 2478452,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983274.082, "dur": 0.390,
    "args": {
      "External id": 2478453,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983292.690, "dur": 44.824,
    "args": {
      "External id": 2478454,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983371.249, "dur": 116.164,
    "args": {
      "External id": 2478455,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983384.033, "dur": 3.851,
    "args": {
      "External id": 2478456,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983392.921, "dur": 10.332,
    "args": {
      "External id": 2478457,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983397.829, "dur": 4.989,
    "args": {
      "External id": 2478458,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983400.939, "dur": 0.735,
    "args": {
      "External id": 2478459,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983409.619, "dur": 36.360,
    "args": {
      "External id": 2478460,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983413.406, "dur": 3.000,
    "args": {
      "External id": 2478461,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983417.230, "dur": 0.811,
    "args": {
      "External id": 2478462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983418.924, "dur": 0.565,
    "args": {
      "External id": 2478463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983424.355, "dur": 2.273,
    "args": {
      "External id": 2478464,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983427.559, "dur": 0.334,
    "args": {
      "External id": 2478465,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983430.750, "dur": 0.458,
    "args": {
      "External id": 2478466,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983434.077, "dur": 0.656,
    "args": {
      "External id": 2478467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983435.712, "dur": 0.363,
    "args": {
      "External id": 2478468,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983438.978, "dur": 2.235,
    "args": {
      "External id": 2478469,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983456.380, "dur": 22.768,
    "args": {
      "External id": 2478470,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983559.216, "dur": 380.095,
    "args": {
      "External id": 2478471,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983592.615, "dur": 341.236,
    "args": {
      "External id": 2478472,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6175, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983606.016, "dur": 321.571,
    "args": {
      "External id": 2478473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514655983966.468, "dur": 2.176,
    "args": {
      "External id": 2478474,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6177, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984097.844, "dur": 16788.560,
    "args": {
      "External id": 2478475,"Record function id": 0, "Ev Idx": 6178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984201.648, "dur": 6.631,
    "args": {
      "External id": 2478476,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984211.656, "dur": 1.151,
    "args": {
      "External id": 2478477,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984214.573, "dur": 2.550,
    "args": {
      "External id": 2478478,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984218.728, "dur": 1.636,
    "args": {
      "External id": 2478479,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984221.798, "dur": 1.042,
    "args": {
      "External id": 2478480,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984224.514, "dur": 0.684,
    "args": {
      "External id": 2478481,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984228.730, "dur": 1.134,
    "args": {
      "External id": 2478482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984231.477, "dur": 2.172,
    "args": {
      "External id": 2478483,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984235.290, "dur": 1.001,
    "args": {
      "External id": 2478484,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984237.584, "dur": 0.867,
    "args": {
      "External id": 2478485,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984258.885, "dur": 16589.243,
    "args": {
      "External id": 2478486,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984274.897, "dur": 16566.715,
    "args": {
      "External id": 2478487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984297.891, "dur": 14.668,
    "args": {
      "External id": 2478488,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984315.773, "dur": 16492.044,
    "args": {
      "External id": 2478489,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984318.189, "dur": 16489.097,
    "args": {
      "External id": 2478490,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984324.361, "dur": 4.817,
    "args": {
      "External id": 2478491,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514655984330.952, "dur": 16473.559,
    "args": {
      "External id": 2478492,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001095.015, "dur": 32.231,
    "args": {
      "External id": 2478493,"Sequence number": 24740425, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6196
    }
  },
  {
    "ph": "s", "id": 223, "pid": 1336760, "tid": 1336760, "ts": 1514656001095.015,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001114.086, "dur": 8.149,
    "args": {
      "External id": 2478494,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001117.324, "dur": 4.575,
    "args": {
      "External id": 2478495,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001196.102, "dur": 85.452,
    "args": {
      "External id": 2478496,"Record function id": 0, "Ev Idx": 6199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001282.938, "dur": 1119.475,
    "args": {
      "External id": 2478497,"Record function id": 0, "Ev Idx": 6200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001324.799, "dur": 1064.799,
    "args": {
      "External id": 2478498,"Sequence number": 24740426, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6201
    }
  },
  {
    "ph": "s", "id": 222, "pid": 1336760, "tid": 1336760, "ts": 1514656001324.799,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001391.787, "dur": 46.421,
    "args": {
      "External id": 2478499,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001450.381, "dur": 120.913,
    "args": {
      "External id": 2478500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001584.325, "dur": 43.558,
    "args": {
      "External id": 2478501,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001637.364, "dur": 32.476,
    "args": {
      "External id": 2478502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001694.713, "dur": 28.706,
    "args": {
      "External id": 2478503,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001740.418, "dur": 16.368,
    "args": {
      "External id": 2478504,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001775.648, "dur": 132.186,
    "args": {
      "External id": 2478505,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001825.139, "dur": 12.845,
    "args": {
      "External id": 2478506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001832.054, "dur": 4.860,
    "args": {
      "External id": 2478507,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001840.546, "dur": 6.458,
    "args": {
      "External id": 2478508,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001848.136, "dur": 1.048,
    "args": {
      "External id": 2478509,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001851.360, "dur": 4.598,
    "args": {
      "External id": 2478510,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656001918.814, "dur": 47.100,
    "args": {
      "External id": 2478511,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002042.120, "dur": 33.975,
    "args": {
      "External id": 2478512,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002087.597, "dur": 46.633,
    "args": {
      "External id": 2478513,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002142.926, "dur": 36.917,
    "args": {
      "External id": 2478514,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002201.349, "dur": 28.905,
    "args": {
      "External id": 2478515,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002236.319, "dur": 34.751,
    "args": {
      "External id": 2478516,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002293.419, "dur": 20.520,
    "args": {
      "External id": 2478517,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6220
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.2)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002465.928, "dur": 107.930,
    "args": {
      "External id": 2478518,"Record function id": 0, "Ev Idx": 6221
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002650.321, "dur": 47.741,
    "args": {
      "External id": 2478519,"Record function id": 0, "Ev Idx": 6222
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.3)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002708.322, "dur": 18962.920,
    "args": {
      "External id": 2478520,"Record function id": 0, "Ev Idx": 6223
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002718.864, "dur": 1047.728,
    "args": {
      "External id": 2478521,"Record function id": 0, "Ev Idx": 6224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002802.065, "dur": 8.927,
    "args": {
      "External id": 2478522,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002824.232, "dur": 46.631,
    "args": {
      "External id": 2478523,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002829.361, "dur": 2.486,
    "args": {
      "External id": 2478524,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002839.059, "dur": 0.698,
    "args": {
      "External id": 2478525,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002841.065, "dur": 0.442,
    "args": {
      "External id": 2478526,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002846.560, "dur": 0.805,
    "args": {
      "External id": 2478527,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002851.728, "dur": 0.811,
    "args": {
      "External id": 2478528,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002853.490, "dur": 0.758,
    "args": {
      "External id": 2478529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002856.943, "dur": 4.314,
    "args": {
      "External id": 2478530,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002862.042, "dur": 0.635,
    "args": {
      "External id": 2478531,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002863.557, "dur": 0.410,
    "args": {
      "External id": 2478532,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002885.005, "dur": 42.614,
    "args": {
      "External id": 2478533,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002959.527, "dur": 188.241,
    "args": {
      "External id": 2478534,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656002972.405, "dur": 3.671,
    "args": {
      "External id": 2478535,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003022.114, "dur": 11.928,
    "args": {
      "External id": 2478536,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003027.135, "dur": 6.495,
    "args": {
      "External id": 2478537,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003030.885, "dur": 0.972,
    "args": {
      "External id": 2478538,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003045.659, "dur": 38.065,
    "args": {
      "External id": 2478539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003047.403, "dur": 3.030,
    "args": {
      "External id": 2478540,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003051.551, "dur": 0.743,
    "args": {
      "External id": 2478541,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003056.424, "dur": 0.354,
    "args": {
      "External id": 2478542,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003060.114, "dur": 1.966,
    "args": {
      "External id": 2478543,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003064.854, "dur": 0.574,
    "args": {
      "External id": 2478544,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003066.633, "dur": 0.331,
    "args": {
      "External id": 2478545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003069.833, "dur": 0.798,
    "args": {
      "External id": 2478546,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003073.872, "dur": 0.595,
    "args": {
      "External id": 2478547,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003075.307, "dur": 2.961,
    "args": {
      "External id": 2478548,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003108.771, "dur": 29.599,
    "args": {
      "External id": 2478549,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003202.374, "dur": 460.709,
    "args": {
      "External id": 2478550,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003235.402, "dur": 421.406,
    "args": {
      "External id": 2478551,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6254, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003251.812, "dur": 395.613,
    "args": {
      "External id": 2478552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003690.633, "dur": 2.337,
    "args": {
      "External id": 2478553,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6256, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003788.119, "dur": 17662.719,
    "args": {
      "External id": 2478554,"Record function id": 0, "Ev Idx": 6257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003894.939, "dur": 6.362,
    "args": {
      "External id": 2478555,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003904.919, "dur": 1.293,
    "args": {
      "External id": 2478556,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003908.152, "dur": 2.649,
    "args": {
      "External id": 2478557,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003912.667, "dur": 1.316,
    "args": {
      "External id": 2478558,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003915.672, "dur": 1.217,
    "args": {
      "External id": 2478559,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003918.525, "dur": 0.907,
    "args": {
      "External id": 2478560,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003923.141, "dur": 0.896,
    "args": {
      "External id": 2478561,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003925.707, "dur": 2.462,
    "args": {
      "External id": 2478562,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003929.401, "dur": 0.950,
    "args": {
      "External id": 2478563,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003931.718, "dur": 0.934,
    "args": {
      "External id": 2478564,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003955.300, "dur": 17455.502,
    "args": {
      "External id": 2478565,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656003971.742, "dur": 17432.398,
    "args": {
      "External id": 2478566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656004034.859, "dur": 15.462,
    "args": {
      "External id": 2478567,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656004053.527, "dur": 17314.869,
    "args": {
      "External id": 2478568,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656004056.343, "dur": 17311.381,
    "args": {
      "External id": 2478569,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656004065.240, "dur": 7.011,
    "args": {
      "External id": 2478570,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656004074.140, "dur": 17290.776,
    "args": {
      "External id": 2478571,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021605.715, "dur": 39.772,
    "args": {
      "External id": 2478572,"Sequence number": 24740427, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6275
    }
  },
  {
    "ph": "s", "id": 221, "pid": 1336760, "tid": 1336760, "ts": 1514656021605.715,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021632.322, "dur": 8.287,
    "args": {
      "External id": 2478573,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021635.759, "dur": 4.516,
    "args": {
      "External id": 2478574,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021712.669, "dur": 82.975,
    "args": {
      "External id": 2478575,"Record function id": 0, "Ev Idx": 6278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021799.639, "dur": 1139.247,
    "args": {
      "External id": 2478576,"Record function id": 0, "Ev Idx": 6279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021841.432, "dur": 1084.068,
    "args": {
      "External id": 2478577,"Sequence number": 24740428, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6280
    }
  },
  {
    "ph": "s", "id": 220, "pid": 1336760, "tid": 1336760, "ts": 1514656021841.432,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021906.757, "dur": 44.468,
    "args": {
      "External id": 2478578,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656021962.401, "dur": 132.935,
    "args": {
      "External id": 2478579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022113.738, "dur": 41.862,
    "args": {
      "External id": 2478580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022161.467, "dur": 31.752,
    "args": {
      "External id": 2478581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022223.661, "dur": 29.568,
    "args": {
      "External id": 2478582,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022270.421, "dur": 17.018,
    "args": {
      "External id": 2478583,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022306.657, "dur": 134.641,
    "args": {
      "External id": 2478584,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022360.377, "dur": 10.506,
    "args": {
      "External id": 2478585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022365.176, "dur": 4.983,
    "args": {
      "External id": 2478586,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022373.705, "dur": 4.951,
    "args": {
      "External id": 2478587,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022380.011, "dur": 1.272,
    "args": {
      "External id": 2478588,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022386.084, "dur": 4.702,
    "args": {
      "External id": 2478589,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022452.396, "dur": 46.233,
    "args": {
      "External id": 2478590,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022547.865, "dur": 36.528,
    "args": {
      "External id": 2478591,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022597.349, "dur": 46.149,
    "args": {
      "External id": 2478592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022678.085, "dur": 36.878,
    "args": {
      "External id": 2478593,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022745.356, "dur": 30.219,
    "args": {
      "External id": 2478594,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022781.542, "dur": 34.398,
    "args": {
      "External id": 2478595,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656022833.436, "dur": 25.368,
    "args": {
      "External id": 2478596,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6299
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.3)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023038.821, "dur": 81.844,
    "args": {
      "External id": 2478597,"Record function id": 0, "Ev Idx": 6300
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023196.986, "dur": 48.081,
    "args": {
      "External id": 2478598,"Record function id": 0, "Ev Idx": 6301
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.4)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023255.345, "dur": 17808.344,
    "args": {
      "External id": 2478599,"Record function id": 0, "Ev Idx": 6302
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023267.392, "dur": 952.184,
    "args": {
      "External id": 2478600,"Record function id": 0, "Ev Idx": 6303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023351.498, "dur": 8.400,
    "args": {
      "External id": 2478601,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023373.288, "dur": 42.060,
    "args": {
      "External id": 2478602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023380.258, "dur": 2.303,
    "args": {
      "External id": 2478603,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023386.532, "dur": 0.418,
    "args": {
      "External id": 2478604,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023387.785, "dur": 0.406,
    "args": {
      "External id": 2478605,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023391.688, "dur": 0.540,
    "args": {
      "External id": 2478606,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023395.350, "dur": 0.636,
    "args": {
      "External id": 2478607,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023398.714, "dur": 0.638,
    "args": {
      "External id": 2478608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023399.965, "dur": 3.146,
    "args": {
      "External id": 2478609,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023403.902, "dur": 0.321,
    "args": {
      "External id": 2478610,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023407.025, "dur": 0.369,
    "args": {
      "External id": 2478611,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023426.751, "dur": 44.587,
    "args": {
      "External id": 2478612,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023502.081, "dur": 143.370,
    "args": {
      "External id": 2478613,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023514.826, "dur": 3.556,
    "args": {
      "External id": 2478614,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023523.863, "dur": 29.944,
    "args": {
      "External id": 2478615,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023545.566, "dur": 7.817,
    "args": {
      "External id": 2478616,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023550.711, "dur": 0.922,
    "args": {
      "External id": 2478617,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023561.692, "dur": 35.859,
    "args": {
      "External id": 2478618,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023563.563, "dur": 2.778,
    "args": {
      "External id": 2478619,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023569.615, "dur": 0.639,
    "args": {
      "External id": 2478620,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023570.848, "dur": 0.396,
    "args": {
      "External id": 2478621,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023576.207, "dur": 3.281,
    "args": {
      "External id": 2478622,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023580.154, "dur": 1.020,
    "args": {
      "External id": 2478623,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023581.988, "dur": 0.378,
    "args": {
      "External id": 2478624,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023586.922, "dur": 0.338,
    "args": {
      "External id": 2478625,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023588.277, "dur": 0.403,
    "args": {
      "External id": 2478626,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023589.421, "dur": 2.741,
    "args": {
      "External id": 2478627,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023611.266, "dur": 25.602,
    "args": {
      "External id": 2478628,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023698.508, "dur": 419.920,
    "args": {
      "External id": 2478629,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023732.118, "dur": 380.458,
    "args": {
      "External id": 2478630,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6333, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656023744.800, "dur": 361.195,
    "args": {
      "External id": 2478631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024146.218, "dur": 2.342,
    "args": {
      "External id": 2478632,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6335, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024239.827, "dur": 16596.674,
    "args": {
      "External id": 2478633,"Record function id": 0, "Ev Idx": 6336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024343.926, "dur": 6.374,
    "args": {
      "External id": 2478634,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024353.551, "dur": 1.076,
    "args": {
      "External id": 2478635,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024356.467, "dur": 3.208,
    "args": {
      "External id": 2478636,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024361.280, "dur": 1.052,
    "args": {
      "External id": 2478637,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024363.892, "dur": 1.216,
    "args": {
      "External id": 2478638,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024366.320, "dur": 1.183,
    "args": {
      "External id": 2478639,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024371.155, "dur": 1.019,
    "args": {
      "External id": 2478640,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024375.715, "dur": 3.466,
    "args": {
      "External id": 2478641,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024380.529, "dur": 0.866,
    "args": {
      "External id": 2478642,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024382.841, "dur": 0.932,
    "args": {
      "External id": 2478643,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024402.650, "dur": 16395.600,
    "args": {
      "External id": 2478644,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024419.007, "dur": 16372.183,
    "args": {
      "External id": 2478645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024441.704, "dur": 13.465,
    "args": {
      "External id": 2478646,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024458.201, "dur": 16296.818,
    "args": {
      "External id": 2478647,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024460.831, "dur": 16293.624,
    "args": {
      "External id": 2478648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024466.434, "dur": 4.951,
    "args": {
      "External id": 2478649,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656024473.073, "dur": 16278.557,
    "args": {
      "External id": 2478650,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656040975.724, "dur": 61.975,
    "args": {
      "External id": 2478651,"Sequence number": 24740429, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6354
    }
  },
  {
    "ph": "s", "id": 219, "pid": 1336760, "tid": 1336760, "ts": 1514656040975.724,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041021.391, "dur": 11.178,
    "args": {
      "External id": 2478652,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041027.235, "dur": 4.990,
    "args": {
      "External id": 2478653,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041106.016, "dur": 81.078,
    "args": {
      "External id": 2478654,"Record function id": 0, "Ev Idx": 6357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041188.648, "dur": 1151.747,
    "args": {
      "External id": 2478655,"Record function id": 0, "Ev Idx": 6358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041228.122, "dur": 1098.638,
    "args": {
      "External id": 2478656,"Sequence number": 24740430, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6359
    }
  },
  {
    "ph": "s", "id": 218, "pid": 1336760, "tid": 1336760, "ts": 1514656041228.122,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041295.902, "dur": 45.547,
    "args": {
      "External id": 2478657,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041353.902, "dur": 105.868,
    "args": {
      "External id": 2478658,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041469.817, "dur": 39.494,
    "args": {
      "External id": 2478659,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041518.538, "dur": 56.804,
    "args": {
      "External id": 2478660,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041604.010, "dur": 32.653,
    "args": {
      "External id": 2478661,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041656.408, "dur": 17.805,
    "args": {
      "External id": 2478662,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041693.563, "dur": 132.652,
    "args": {
      "External id": 2478663,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041744.063, "dur": 9.900,
    "args": {
      "External id": 2478664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041748.333, "dur": 4.903,
    "args": {
      "External id": 2478665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041756.532, "dur": 6.132,
    "args": {
      "External id": 2478666,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041766.295, "dur": 1.224,
    "args": {
      "External id": 2478667,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041769.993, "dur": 4.442,
    "args": {
      "External id": 2478668,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041836.178, "dur": 50.749,
    "args": {
      "External id": 2478669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041919.390, "dur": 31.292,
    "args": {
      "External id": 2478670,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656041960.178, "dur": 89.410,
    "args": {
      "External id": 2478671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042061.952, "dur": 38.838,
    "args": {
      "External id": 2478672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042125.822, "dur": 34.063,
    "args": {
      "External id": 2478673,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042165.903, "dur": 36.116,
    "args": {
      "External id": 2478674,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042222.839, "dur": 22.606,
    "args": {
      "External id": 2478675,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6378
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.4)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042403.973, "dur": 80.974,
    "args": {
      "External id": 2478676,"Record function id": 0, "Ev Idx": 6379
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042582.765, "dur": 47.793,
    "args": {
      "External id": 2478677,"Record function id": 0, "Ev Idx": 6380
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.5)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042640.656, "dur": 18230.527,
    "args": {
      "External id": 2478678,"Record function id": 0, "Ev Idx": 6381
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042651.378, "dur": 964.590,
    "args": {
      "External id": 2478679,"Record function id": 0, "Ev Idx": 6382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042739.321, "dur": 8.384,
    "args": {
      "External id": 2478680,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042760.989, "dur": 46.219,
    "args": {
      "External id": 2478681,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042768.289, "dur": 2.185,
    "args": {
      "External id": 2478682,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042774.582, "dur": 0.441,
    "args": {
      "External id": 2478683,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042777.780, "dur": 0.687,
    "args": {
      "External id": 2478684,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042779.180, "dur": 0.492,
    "args": {
      "External id": 2478685,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042783.110, "dur": 0.601,
    "args": {
      "External id": 2478686,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042787.019, "dur": 0.679,
    "args": {
      "External id": 2478687,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042788.353, "dur": 5.806,
    "args": {
      "External id": 2478688,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042794.910, "dur": 0.344,
    "args": {
      "External id": 2478689,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042798.219, "dur": 0.543,
    "args": {
      "External id": 2478690,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042820.018, "dur": 43.711,
    "args": {
      "External id": 2478691,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042895.645, "dur": 166.246,
    "args": {
      "External id": 2478692,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042910.903, "dur": 3.762,
    "args": {
      "External id": 2478693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042920.109, "dur": 9.394,
    "args": {
      "External id": 2478694,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042924.600, "dur": 4.478,
    "args": {
      "External id": 2478695,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042927.287, "dur": 0.686,
    "args": {
      "External id": 2478696,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042936.039, "dur": 38.897,
    "args": {
      "External id": 2478697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042939.791, "dur": 2.934,
    "args": {
      "External id": 2478698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042943.664, "dur": 0.390,
    "args": {
      "External id": 2478699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042945.140, "dur": 0.380,
    "args": {
      "External id": 2478700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042951.134, "dur": 2.204,
    "args": {
      "External id": 2478701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042954.250, "dur": 0.559,
    "args": {
      "External id": 2478702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042955.865, "dur": 0.685,
    "args": {
      "External id": 2478703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042962.261, "dur": 0.561,
    "args": {
      "External id": 2478704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042963.485, "dur": 0.484,
    "args": {
      "External id": 2478705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656042966.598, "dur": 2.589,
    "args": {
      "External id": 2478706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043026.139, "dur": 26.309,
    "args": {
      "External id": 2478707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043117.556, "dur": 383.816,
    "args": {
      "External id": 2478708,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043151.189, "dur": 345.117,
    "args": {
      "External id": 2478709,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6412, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043163.352, "dur": 327.130,
    "args": {
      "External id": 2478710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043525.782, "dur": 19.194,
    "args": {
      "External id": 2478711,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6414, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043636.014, "dur": 17037.621,
    "args": {
      "External id": 2478712,"Record function id": 0, "Ev Idx": 6415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043740.370, "dur": 6.864,
    "args": {
      "External id": 2478713,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043750.531, "dur": 1.209,
    "args": {
      "External id": 2478714,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043753.450, "dur": 3.389,
    "args": {
      "External id": 2478715,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043758.224, "dur": 1.383,
    "args": {
      "External id": 2478716,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043761.153, "dur": 1.162,
    "args": {
      "External id": 2478717,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043763.847, "dur": 1.248,
    "args": {
      "External id": 2478718,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043769.089, "dur": 1.273,
    "args": {
      "External id": 2478719,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043772.160, "dur": 3.017,
    "args": {
      "External id": 2478720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043776.919, "dur": 0.814,
    "args": {
      "External id": 2478721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043779.074, "dur": 0.996,
    "args": {
      "External id": 2478722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043799.525, "dur": 16831.805,
    "args": {
      "External id": 2478723,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043819.568, "dur": 16805.169,
    "args": {
      "External id": 2478724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043840.819, "dur": 15.584,
    "args": {
      "External id": 2478725,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043859.437, "dur": 16733.180,
    "args": {
      "External id": 2478726,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043862.276, "dur": 16729.810,
    "args": {
      "External id": 2478727,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043868.568, "dur": 4.655,
    "args": {
      "External id": 2478728,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656043875.062, "dur": 16714.237,
    "args": {
      "External id": 2478729,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656060819.687, "dur": 28.652,
    "args": {
      "External id": 2478730,"Sequence number": 24740431, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6433
    }
  },
  {
    "ph": "s", "id": 217, "pid": 1336760, "tid": 1336760, "ts": 1514656060819.687,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656060835.790, "dur": 7.739,
    "args": {
      "External id": 2478731,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656060839.119, "dur": 4.218,
    "args": {
      "External id": 2478732,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656060913.958, "dur": 115.731,
    "args": {
      "External id": 2478733,"Record function id": 0, "Ev Idx": 6436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061033.254, "dur": 1123.820,
    "args": {
      "External id": 2478734,"Record function id": 0, "Ev Idx": 6437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061081.033, "dur": 1062.838,
    "args": {
      "External id": 2478735,"Sequence number": 24740432, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6438
    }
  },
  {
    "ph": "s", "id": 216, "pid": 1336760, "tid": 1336760, "ts": 1514656061081.033,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061149.071, "dur": 47.172,
    "args": {
      "External id": 2478736,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061209.943, "dur": 99.719,
    "args": {
      "External id": 2478737,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061321.988, "dur": 40.540,
    "args": {
      "External id": 2478738,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061371.120, "dur": 31.209,
    "args": {
      "External id": 2478739,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061427.184, "dur": 27.762,
    "args": {
      "External id": 2478740,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061474.409, "dur": 18.185,
    "args": {
      "External id": 2478741,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061517.971, "dur": 154.404,
    "args": {
      "External id": 2478742,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061587.415, "dur": 12.500,
    "args": {
      "External id": 2478743,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061592.248, "dur": 6.652,
    "args": {
      "External id": 2478744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061603.067, "dur": 5.164,
    "args": {
      "External id": 2478745,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061609.300, "dur": 0.856,
    "args": {
      "External id": 2478746,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061612.724, "dur": 6.003,
    "args": {
      "External id": 2478747,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061683.964, "dur": 51.952,
    "args": {
      "External id": 2478748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061770.041, "dur": 33.417,
    "args": {
      "External id": 2478749,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061813.537, "dur": 42.528,
    "args": {
      "External id": 2478750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061867.962, "dur": 35.284,
    "args": {
      "External id": 2478751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061923.546, "dur": 29.974,
    "args": {
      "External id": 2478752,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656061959.437, "dur": 67.414,
    "args": {
      "External id": 2478753,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062053.880, "dur": 20.431,
    "args": {
      "External id": 2478754,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6457
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.5)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062220.276, "dur": 79.799,
    "args": {
      "External id": 2478755,"Record function id": 0, "Ev Idx": 6458
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062373.511, "dur": 44.272,
    "args": {
      "External id": 2478756,"Record function id": 0, "Ev Idx": 6459
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.6)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062427.578, "dur": 18038.555,
    "args": {
      "External id": 2478757,"Record function id": 0, "Ev Idx": 6460
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062435.746, "dur": 935.760,
    "args": {
      "External id": 2478758,"Record function id": 0, "Ev Idx": 6461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062519.429, "dur": 22.954,
    "args": {
      "External id": 2478759,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062557.799, "dur": 44.910,
    "args": {
      "External id": 2478760,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062563.680, "dur": 2.477,
    "args": {
      "External id": 2478761,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062570.941, "dur": 0.528,
    "args": {
      "External id": 2478762,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062573.657, "dur": 0.398,
    "args": {
      "External id": 2478763,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062576.106, "dur": 0.561,
    "args": {
      "External id": 2478764,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062580.281, "dur": 0.520,
    "args": {
      "External id": 2478765,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062582.512, "dur": 0.616,
    "args": {
      "External id": 2478766,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062585.035, "dur": 4.786,
    "args": {
      "External id": 2478767,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062591.631, "dur": 0.670,
    "args": {
      "External id": 2478768,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062594.259, "dur": 0.347,
    "args": {
      "External id": 2478769,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062616.041, "dur": 39.656,
    "args": {
      "External id": 2478770,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062691.667, "dur": 127.891,
    "args": {
      "External id": 2478771,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062705.061, "dur": 4.824,
    "args": {
      "External id": 2478772,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062715.517, "dur": 10.399,
    "args": {
      "External id": 2478773,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062720.266, "dur": 5.248,
    "args": {
      "External id": 2478774,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062723.734, "dur": 0.677,
    "args": {
      "External id": 2478775,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062732.897, "dur": 42.887,
    "args": {
      "External id": 2478776,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062736.120, "dur": 2.584,
    "args": {
      "External id": 2478777,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062740.053, "dur": 4.973,
    "args": {
      "External id": 2478778,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062746.479, "dur": 0.708,
    "args": {
      "External id": 2478779,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062752.014, "dur": 2.756,
    "args": {
      "External id": 2478780,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062756.587, "dur": 0.556,
    "args": {
      "External id": 2478781,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062759.077, "dur": 0.542,
    "args": {
      "External id": 2478782,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062763.315, "dur": 0.626,
    "args": {
      "External id": 2478783,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062765.504, "dur": 0.644,
    "args": {
      "External id": 2478784,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062767.518, "dur": 2.639,
    "args": {
      "External id": 2478785,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062787.306, "dur": 23.941,
    "args": {
      "External id": 2478786,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062873.926, "dur": 399.943,
    "args": {
      "External id": 2478787,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062907.805, "dur": 360.366,
    "args": {
      "External id": 2478788,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6491, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656062919.963, "dur": 341.788,
    "args": {
      "External id": 2478789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063300.299, "dur": 2.460,
    "args": {
      "External id": 2478790,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6493, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063391.407, "dur": 16878.998,
    "args": {
      "External id": 2478791,"Record function id": 0, "Ev Idx": 6494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063493.066, "dur": 6.545,
    "args": {
      "External id": 2478792,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063503.858, "dur": 1.404,
    "args": {
      "External id": 2478793,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063507.189, "dur": 3.266,
    "args": {
      "External id": 2478794,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063512.303, "dur": 1.041,
    "args": {
      "External id": 2478795,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063514.952, "dur": 1.645,
    "args": {
      "External id": 2478796,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063518.303, "dur": 1.314,
    "args": {
      "External id": 2478797,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063524.189, "dur": 0.932,
    "args": {
      "External id": 2478798,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063544.569, "dur": 4.390,
    "args": {
      "External id": 2478799,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063552.387, "dur": 1.144,
    "args": {
      "External id": 2478800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063555.188, "dur": 0.847,
    "args": {
      "External id": 2478801,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063578.924, "dur": 16643.782,
    "args": {
      "External id": 2478802,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063595.158, "dur": 16621.075,
    "args": {
      "External id": 2478803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063619.750, "dur": 15.101,
    "args": {
      "External id": 2478804,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063637.876, "dur": 16544.030,
    "args": {
      "External id": 2478805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063640.742, "dur": 16540.649,
    "args": {
      "External id": 2478806,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063647.190, "dur": 5.361,
    "args": {
      "External id": 2478807,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656063654.595, "dur": 16524.203,
    "args": {
      "External id": 2478808,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080414.113, "dur": 28.023,
    "args": {
      "External id": 2478809,"Sequence number": 24740433, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6512
    }
  },
  {
    "ph": "s", "id": 215, "pid": 1336760, "tid": 1336760, "ts": 1514656080414.113,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080429.637, "dur": 8.025,
    "args": {
      "External id": 2478810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080433.019, "dur": 4.473,
    "args": {
      "External id": 2478811,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080508.206, "dur": 94.318,
    "args": {
      "External id": 2478812,"Record function id": 0, "Ev Idx": 6515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080605.267, "dur": 1113.608,
    "args": {
      "External id": 2478813,"Record function id": 0, "Ev Idx": 6516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080649.530, "dur": 1056.531,
    "args": {
      "External id": 2478814,"Sequence number": 24740434, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6517
    }
  },
  {
    "ph": "s", "id": 214, "pid": 1336760, "tid": 1336760, "ts": 1514656080649.530,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080715.223, "dur": 46.713,
    "args": {
      "External id": 2478815,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080774.661, "dur": 104.525,
    "args": {
      "External id": 2478816,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080889.073, "dur": 39.769,
    "args": {
      "External id": 2478817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656080939.271, "dur": 32.753,
    "args": {
      "External id": 2478818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081031.951, "dur": 31.053,
    "args": {
      "External id": 2478819,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081083.067, "dur": 16.599,
    "args": {
      "External id": 2478820,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081119.129, "dur": 133.616,
    "args": {
      "External id": 2478821,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081169.269, "dur": 13.720,
    "args": {
      "External id": 2478822,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081174.321, "dur": 7.921,
    "args": {
      "External id": 2478823,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081186.554, "dur": 5.961,
    "args": {
      "External id": 2478824,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081194.047, "dur": 1.055,
    "args": {
      "External id": 2478825,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081197.869, "dur": 4.006,
    "args": {
      "External id": 2478826,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081263.322, "dur": 51.384,
    "args": {
      "External id": 2478827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081346.300, "dur": 32.244,
    "args": {
      "External id": 2478828,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081388.615, "dur": 41.361,
    "args": {
      "External id": 2478829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081439.212, "dur": 36.069,
    "args": {
      "External id": 2478830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081497.977, "dur": 45.079,
    "args": {
      "External id": 2478831,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081551.011, "dur": 40.080,
    "args": {
      "External id": 2478832,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081613.321, "dur": 20.630,
    "args": {
      "External id": 2478833,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.6)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081782.260, "dur": 77.595,
    "args": {
      "External id": 2478834,"Record function id": 0, "Ev Idx": 6537
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656081933.764, "dur": 45.211,
    "args": {
      "External id": 2478835,"Record function id": 0, "Ev Idx": 6538
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.7)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082023.670, "dur": 17928.981,
    "args": {
      "External id": 2478836,"Record function id": 0, "Ev Idx": 6539
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082032.381, "dur": 891.638,
    "args": {
      "External id": 2478837,"Record function id": 0, "Ev Idx": 6540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082116.141, "dur": 8.941,
    "args": {
      "External id": 2478838,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082139.362, "dur": 45.285,
    "args": {
      "External id": 2478839,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082145.342, "dur": 2.467,
    "args": {
      "External id": 2478840,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082152.888, "dur": 0.381,
    "args": {
      "External id": 2478841,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082155.183, "dur": 0.658,
    "args": {
      "External id": 2478842,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082157.502, "dur": 0.703,
    "args": {
      "External id": 2478843,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082162.661, "dur": 0.557,
    "args": {
      "External id": 2478844,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082164.584, "dur": 0.571,
    "args": {
      "External id": 2478845,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082166.807, "dur": 4.969,
    "args": {
      "External id": 2478846,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082173.438, "dur": 0.572,
    "args": {
      "External id": 2478847,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082175.856, "dur": 0.761,
    "args": {
      "External id": 2478848,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082197.190, "dur": 41.841,
    "args": {
      "External id": 2478849,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082272.713, "dur": 122.810,
    "args": {
      "External id": 2478850,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082285.643, "dur": 3.299,
    "args": {
      "External id": 2478851,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082294.417, "dur": 10.624,
    "args": {
      "External id": 2478852,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082298.901, "dur": 5.719,
    "args": {
      "External id": 2478853,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082302.527, "dur": 0.774,
    "args": {
      "External id": 2478854,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082311.820, "dur": 37.403,
    "args": {
      "External id": 2478855,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082314.519, "dur": 2.435,
    "args": {
      "External id": 2478856,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082318.859, "dur": 0.389,
    "args": {
      "External id": 2478857,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082321.456, "dur": 0.600,
    "args": {
      "External id": 2478858,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082326.308, "dur": 2.263,
    "args": {
      "External id": 2478859,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082330.113, "dur": 0.671,
    "args": {
      "External id": 2478860,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082332.537, "dur": 0.562,
    "args": {
      "External id": 2478861,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082336.604, "dur": 0.412,
    "args": {
      "External id": 2478862,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082338.278, "dur": 0.659,
    "args": {
      "External id": 2478863,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082340.366, "dur": 3.089,
    "args": {
      "External id": 2478864,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082362.217, "dur": 24.956,
    "args": {
      "External id": 2478865,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082449.566, "dur": 374.848,
    "args": {
      "External id": 2478866,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082482.967, "dur": 336.074,
    "args": {
      "External id": 2478867,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6570, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082495.224, "dur": 318.251,
    "args": {
      "External id": 2478868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082852.519, "dur": 2.333,
    "args": {
      "External id": 2478869,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6572, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656082944.584, "dur": 16809.799,
    "args": {
      "External id": 2478870,"Record function id": 0, "Ev Idx": 6573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083092.315, "dur": 6.861,
    "args": {
      "External id": 2478871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083103.928, "dur": 1.195,
    "args": {
      "External id": 2478872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083107.094, "dur": 3.078,
    "args": {
      "External id": 2478873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083113.350, "dur": 1.097,
    "args": {
      "External id": 2478874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083115.915, "dur": 1.008,
    "args": {
      "External id": 2478875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083118.431, "dur": 1.166,
    "args": {
      "External id": 2478876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083121.236, "dur": 1.120,
    "args": {
      "External id": 2478877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083125.500, "dur": 2.660,
    "args": {
      "External id": 2478878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083129.800, "dur": 0.935,
    "args": {
      "External id": 2478879,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083132.542, "dur": 1.124,
    "args": {
      "External id": 2478880,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083154.278, "dur": 16556.691,
    "args": {
      "External id": 2478881,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083169.031, "dur": 16535.233,
    "args": {
      "External id": 2478882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083189.896, "dur": 15.725,
    "args": {
      "External id": 2478883,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083211.154, "dur": 16459.330,
    "args": {
      "External id": 2478884,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083213.563, "dur": 16456.234,
    "args": {
      "External id": 2478885,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083219.642, "dur": 4.825,
    "args": {
      "External id": 2478886,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656083226.162, "dur": 16440.986,
    "args": {
      "External id": 2478887,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656099898.160, "dur": 29.822,
    "args": {
      "External id": 2478888,"Sequence number": 24740435, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6591
    }
  },
  {
    "ph": "s", "id": 213, "pid": 1336760, "tid": 1336760, "ts": 1514656099898.160,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656099916.010, "dur": 7.279,
    "args": {
      "External id": 2478889,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656099918.914, "dur": 4.210,
    "args": {
      "External id": 2478890,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100033.744, "dur": 84.783,
    "args": {
      "External id": 2478891,"Record function id": 0, "Ev Idx": 6594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100120.901, "dur": 1121.768,
    "args": {
      "External id": 2478892,"Record function id": 0, "Ev Idx": 6595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100164.545, "dur": 1064.855,
    "args": {
      "External id": 2478893,"Sequence number": 24740436, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6596
    }
  },
  {
    "ph": "s", "id": 212, "pid": 1336760, "tid": 1336760, "ts": 1514656100164.545,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100233.934, "dur": 42.855,
    "args": {
      "External id": 2478894,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100289.057, "dur": 110.247,
    "args": {
      "External id": 2478895,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100409.219, "dur": 39.213,
    "args": {
      "External id": 2478896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100458.015, "dur": 31.561,
    "args": {
      "External id": 2478897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100513.750, "dur": 43.504,
    "args": {
      "External id": 2478898,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100578.183, "dur": 19.460,
    "args": {
      "External id": 2478899,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100615.962, "dur": 138.985,
    "args": {
      "External id": 2478900,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100665.399, "dur": 11.691,
    "args": {
      "External id": 2478901,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100670.370, "dur": 5.891,
    "args": {
      "External id": 2478902,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100680.052, "dur": 7.286,
    "args": {
      "External id": 2478903,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100695.924, "dur": 1.094,
    "args": {
      "External id": 2478904,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100699.457, "dur": 4.514,
    "args": {
      "External id": 2478905,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100765.775, "dur": 50.815,
    "args": {
      "External id": 2478906,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100850.928, "dur": 29.173,
    "args": {
      "External id": 2478907,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100890.659, "dur": 42.930,
    "args": {
      "External id": 2478908,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656100941.606, "dur": 36.094,
    "args": {
      "External id": 2478909,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101034.910, "dur": 36.156,
    "args": {
      "External id": 2478910,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101077.738, "dur": 39.114,
    "args": {
      "External id": 2478911,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101139.821, "dur": 18.701,
    "args": {
      "External id": 2478912,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6615
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.7)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101306.579, "dur": 79.573,
    "args": {
      "External id": 2478913,"Record function id": 0, "Ev Idx": 6616
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101462.127, "dur": 45.697,
    "args": {
      "External id": 2478914,"Record function id": 0, "Ev Idx": 6617
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.8)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101516.768, "dur": 17891.856,
    "args": {
      "External id": 2478915,"Record function id": 0, "Ev Idx": 6618
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101526.010, "dur": 914.216,
    "args": {
      "External id": 2478916,"Record function id": 0, "Ev Idx": 6619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101632.231, "dur": 8.516,
    "args": {
      "External id": 2478917,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101655.127, "dur": 43.237,
    "args": {
      "External id": 2478918,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101660.543, "dur": 2.417,
    "args": {
      "External id": 2478919,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101668.104, "dur": 0.636,
    "args": {
      "External id": 2478920,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101670.749, "dur": 0.644,
    "args": {
      "External id": 2478921,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101672.872, "dur": 0.705,
    "args": {
      "External id": 2478922,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101677.293, "dur": 0.470,
    "args": {
      "External id": 2478923,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101679.243, "dur": 0.780,
    "args": {
      "External id": 2478924,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101681.844, "dur": 4.887,
    "args": {
      "External id": 2478925,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101688.653, "dur": 0.857,
    "args": {
      "External id": 2478926,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101691.136, "dur": 0.586,
    "args": {
      "External id": 2478927,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101711.137, "dur": 41.935,
    "args": {
      "External id": 2478928,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101786.091, "dur": 120.812,
    "args": {
      "External id": 2478929,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101799.031, "dur": 3.639,
    "args": {
      "External id": 2478930,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101808.381, "dur": 10.847,
    "args": {
      "External id": 2478931,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101813.236, "dur": 5.569,
    "args": {
      "External id": 2478932,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101816.870, "dur": 0.722,
    "args": {
      "External id": 2478933,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101826.180, "dur": 37.755,
    "args": {
      "External id": 2478934,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101828.978, "dur": 2.668,
    "args": {
      "External id": 2478935,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101833.290, "dur": 0.632,
    "args": {
      "External id": 2478936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101835.650, "dur": 0.406,
    "args": {
      "External id": 2478937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101840.301, "dur": 2.749,
    "args": {
      "External id": 2478938,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101845.122, "dur": 0.773,
    "args": {
      "External id": 2478939,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101847.054, "dur": 0.582,
    "args": {
      "External id": 2478940,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101851.368, "dur": 0.638,
    "args": {
      "External id": 2478941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101853.238, "dur": 0.473,
    "args": {
      "External id": 2478942,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101855.390, "dur": 2.711,
    "args": {
      "External id": 2478943,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101875.472, "dur": 22.909,
    "args": {
      "External id": 2478944,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656101959.592, "dur": 385.751,
    "args": {
      "External id": 2478945,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102027.873, "dur": 312.470,
    "args": {
      "External id": 2478946,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6649, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102041.430, "dur": 293.474,
    "args": {
      "External id": 2478947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102370.228, "dur": 2.097,
    "args": {
      "External id": 2478948,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6651, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102460.254, "dur": 16760.580,
    "args": {
      "External id": 2478949,"Record function id": 0, "Ev Idx": 6652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102580.426, "dur": 6.711,
    "args": {
      "External id": 2478950,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102591.773, "dur": 1.322,
    "args": {
      "External id": 2478951,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102594.973, "dur": 3.455,
    "args": {
      "External id": 2478952,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102600.671, "dur": 1.270,
    "args": {
      "External id": 2478953,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102603.439, "dur": 1.156,
    "args": {
      "External id": 2478954,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102606.038, "dur": 0.839,
    "args": {
      "External id": 2478955,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102610.448, "dur": 1.044,
    "args": {
      "External id": 2478956,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102613.030, "dur": 2.960,
    "args": {
      "External id": 2478957,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102617.913, "dur": 1.504,
    "args": {
      "External id": 2478958,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102620.687, "dur": 1.089,
    "args": {
      "External id": 2478959,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102652.822, "dur": 16524.071,
    "args": {
      "External id": 2478960,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102668.803, "dur": 16501.429,
    "args": {
      "External id": 2478961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102688.673, "dur": 15.463,
    "args": {
      "External id": 2478962,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102707.001, "dur": 16429.207,
    "args": {
      "External id": 2478963,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102709.519, "dur": 16426.121,
    "args": {
      "External id": 2478964,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102715.488, "dur": 4.897,
    "args": {
      "External id": 2478965,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656102722.082, "dur": 16410.844,
    "args": {
      "External id": 2478966,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119359.123, "dur": 26.672,
    "args": {
      "External id": 2478967,"Sequence number": 24740437, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6670
    }
  },
  {
    "ph": "s", "id": 211, "pid": 1336760, "tid": 1336760, "ts": 1514656119359.123,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119374.448, "dur": 6.770,
    "args": {
      "External id": 2478968,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119377.236, "dur": 3.825,
    "args": {
      "External id": 2478969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119447.881, "dur": 95.812,
    "args": {
      "External id": 2478970,"Record function id": 0, "Ev Idx": 6673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119546.826, "dur": 1094.648,
    "args": {
      "External id": 2478971,"Record function id": 0, "Ev Idx": 6674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119588.958, "dur": 1039.381,
    "args": {
      "External id": 2478972,"Sequence number": 24740438, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6675
    }
  },
  {
    "ph": "s", "id": 210, "pid": 1336760, "tid": 1336760, "ts": 1514656119588.958,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119654.701, "dur": 43.164,
    "args": {
      "External id": 2478973,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119709.962, "dur": 103.085,
    "args": {
      "External id": 2478974,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119823.736, "dur": 39.005,
    "args": {
      "External id": 2478975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119872.205, "dur": 31.516,
    "args": {
      "External id": 2478976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119926.632, "dur": 27.737,
    "args": {
      "External id": 2478977,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656119972.180, "dur": 52.030,
    "args": {
      "External id": 2478978,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120046.134, "dur": 130.228,
    "args": {
      "External id": 2478979,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120094.840, "dur": 11.497,
    "args": {
      "External id": 2478980,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120099.619, "dur": 5.867,
    "args": {
      "External id": 2478981,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120109.029, "dur": 7.512,
    "args": {
      "External id": 2478982,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120117.549, "dur": 1.366,
    "args": {
      "External id": 2478983,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120121.173, "dur": 4.829,
    "args": {
      "External id": 2478984,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120187.485, "dur": 53.844,
    "args": {
      "External id": 2478985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120275.004, "dur": 31.619,
    "args": {
      "External id": 2478986,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120316.636, "dur": 41.423,
    "args": {
      "External id": 2478987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120366.679, "dur": 35.714,
    "args": {
      "External id": 2478988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120426.315, "dur": 30.109,
    "args": {
      "External id": 2478989,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120462.137, "dur": 34.628,
    "args": {
      "External id": 2478990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120516.801, "dur": 36.343,
    "args": {
      "External id": 2478991,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6694
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.8)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120707.556, "dur": 81.299,
    "args": {
      "External id": 2478992,"Record function id": 0, "Ev Idx": 6695
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120864.462, "dur": 46.578,
    "args": {
      "External id": 2478993,"Record function id": 0, "Ev Idx": 6696
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.9)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120921.014, "dur": 17896.289,
    "args": {
      "External id": 2478994,"Record function id": 0, "Ev Idx": 6697
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656120928.750, "dur": 949.143,
    "args": {
      "External id": 2478995,"Record function id": 0, "Ev Idx": 6698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121048.179, "dur": 9.341,
    "args": {
      "External id": 2478996,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121071.045, "dur": 41.637,
    "args": {
      "External id": 2478997,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121075.933, "dur": 2.545,
    "args": {
      "External id": 2478998,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121083.080, "dur": 0.673,
    "args": {
      "External id": 2478999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121085.187, "dur": 0.417,
    "args": {
      "External id": 2479000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121086.820, "dur": 0.695,
    "args": {
      "External id": 2479001,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121091.418, "dur": 0.527,
    "args": {
      "External id": 2479002,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121092.850, "dur": 0.593,
    "args": {
      "External id": 2479003,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121094.418, "dur": 4.766,
    "args": {
      "External id": 2479004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121100.998, "dur": 0.582,
    "args": {
      "External id": 2479005,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121102.424, "dur": 0.729,
    "args": {
      "External id": 2479006,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121124.624, "dur": 41.070,
    "args": {
      "External id": 2479007,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121199.247, "dur": 129.017,
    "args": {
      "External id": 2479008,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121212.175, "dur": 3.934,
    "args": {
      "External id": 2479009,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121221.264, "dur": 10.382,
    "args": {
      "External id": 2479010,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121226.039, "dur": 5.181,
    "args": {
      "External id": 2479011,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121229.345, "dur": 0.696,
    "args": {
      "External id": 2479012,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121238.050, "dur": 36.212,
    "args": {
      "External id": 2479013,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121239.620, "dur": 3.152,
    "args": {
      "External id": 2479014,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121247.188, "dur": 0.740,
    "args": {
      "External id": 2479015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121249.143, "dur": 0.563,
    "args": {
      "External id": 2479016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121252.656, "dur": 3.139,
    "args": {
      "External id": 2479017,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121256.761, "dur": 0.441,
    "args": {
      "External id": 2479018,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121257.979, "dur": 2.542,
    "args": {
      "External id": 2479019,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121261.650, "dur": 0.360,
    "args": {
      "External id": 2479020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121263.276, "dur": 0.481,
    "args": {
      "External id": 2479021,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121267.215, "dur": 0.341,
    "args": {
      "External id": 2479022,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121294.484, "dur": 24.929,
    "args": {
      "External id": 2479023,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121383.255, "dur": 394.520,
    "args": {
      "External id": 2479024,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121416.061, "dur": 356.244,
    "args": {
      "External id": 2479025,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6728, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121429.662, "dur": 336.258,
    "args": {
      "External id": 2479026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121805.689, "dur": 2.309,
    "args": {
      "External id": 2479027,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6730, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656121898.201, "dur": 16721.665,
    "args": {
      "External id": 2479028,"Record function id": 0, "Ev Idx": 6731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122051.197, "dur": 7.040,
    "args": {
      "External id": 2479029,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122063.315, "dur": 1.261,
    "args": {
      "External id": 2479030,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122066.342, "dur": 3.650,
    "args": {
      "External id": 2479031,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122071.718, "dur": 1.184,
    "args": {
      "External id": 2479032,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122074.600, "dur": 0.988,
    "args": {
      "External id": 2479033,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122077.249, "dur": 1.198,
    "args": {
      "External id": 2479034,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122082.545, "dur": 0.947,
    "args": {
      "External id": 2479035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122084.785, "dur": 3.928,
    "args": {
      "External id": 2479036,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122090.025, "dur": 1.152,
    "args": {
      "External id": 2479037,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122092.455, "dur": 1.233,
    "args": {
      "External id": 2479038,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122114.645, "dur": 16456.437,
    "args": {
      "External id": 2479039,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122130.829, "dur": 16433.346,
    "args": {
      "External id": 2479040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122152.710, "dur": 14.934,
    "args": {
      "External id": 2479041,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122170.440, "dur": 16348.603,
    "args": {
      "External id": 2479042,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122172.888, "dur": 16345.609,
    "args": {
      "External id": 2479043,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122179.397, "dur": 7.140,
    "args": {
      "External id": 2479044,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656122188.181, "dur": 16327.289,
    "args": {
      "External id": 2479045,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656138764.134, "dur": 29.204,
    "args": {
      "External id": 2479046,"Sequence number": 24740439, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6749
    }
  },
  {
    "ph": "s", "id": 209, "pid": 1336760, "tid": 1336760, "ts": 1514656138764.134,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656138780.687, "dur": 7.944,
    "args": {
      "External id": 2479047,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656138783.857, "dur": 4.550,
    "args": {
      "External id": 2479048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656138858.445, "dur": 80.640,
    "args": {
      "External id": 2479049,"Record function id": 0, "Ev Idx": 6752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656138940.571, "dur": 1143.715,
    "args": {
      "External id": 2479050,"Record function id": 0, "Ev Idx": 6753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139019.668, "dur": 1049.955,
    "args": {
      "External id": 2479051,"Sequence number": 24740440, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6754
    }
  },
  {
    "ph": "s", "id": 208, "pid": 1336760, "tid": 1336760, "ts": 1514656139019.668,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139092.381, "dur": 44.732,
    "args": {
      "External id": 2479052,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139150.437, "dur": 104.964,
    "args": {
      "External id": 2479053,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139265.218, "dur": 37.734,
    "args": {
      "External id": 2479054,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139311.185, "dur": 31.112,
    "args": {
      "External id": 2479055,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139368.145, "dur": 27.647,
    "args": {
      "External id": 2479056,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139414.725, "dur": 16.992,
    "args": {
      "External id": 2479057,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139450.627, "dur": 150.233,
    "args": {
      "External id": 2479058,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139499.453, "dur": 10.673,
    "args": {
      "External id": 2479059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139504.191, "dur": 5.003,
    "args": {
      "External id": 2479060,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139512.845, "dur": 5.710,
    "args": {
      "External id": 2479061,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139519.815, "dur": 1.132,
    "args": {
      "External id": 2479062,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139523.313, "dur": 22.812,
    "args": {
      "External id": 2479063,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139612.257, "dur": 51.673,
    "args": {
      "External id": 2479064,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139693.032, "dur": 30.798,
    "args": {
      "External id": 2479065,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139741.339, "dur": 41.543,
    "args": {
      "External id": 2479066,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139791.633, "dur": 35.165,
    "args": {
      "External id": 2479067,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139852.533, "dur": 28.670,
    "args": {
      "External id": 2479068,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139886.682, "dur": 33.491,
    "args": {
      "External id": 2479069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656139938.520, "dur": 18.395,
    "args": {
      "External id": 2479070,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6773
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.9)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140148.914, "dur": 80.817,
    "args": {
      "External id": 2479071,"Record function id": 0, "Ev Idx": 6774
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140301.735, "dur": 45.081,
    "args": {
      "External id": 2479072,"Record function id": 0, "Ev Idx": 6775
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.10)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140356.521, "dur": 17990.566,
    "args": {
      "External id": 2479073,"Record function id": 0, "Ev Idx": 6776
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140365.776, "dur": 956.664,
    "args": {
      "External id": 2479074,"Record function id": 0, "Ev Idx": 6777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140447.988, "dur": 8.079,
    "args": {
      "External id": 2479075,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140468.755, "dur": 39.597,
    "args": {
      "External id": 2479076,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140473.739, "dur": 2.566,
    "args": {
      "External id": 2479077,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140481.100, "dur": 0.620,
    "args": {
      "External id": 2479078,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140482.758, "dur": 0.542,
    "args": {
      "External id": 2479079,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140484.196, "dur": 0.617,
    "args": {
      "External id": 2479080,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140488.007, "dur": 0.699,
    "args": {
      "External id": 2479081,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140489.596, "dur": 0.638,
    "args": {
      "External id": 2479082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140490.963, "dur": 4.384,
    "args": {
      "External id": 2479083,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140496.236, "dur": 0.585,
    "args": {
      "External id": 2479084,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140499.047, "dur": 0.399,
    "args": {
      "External id": 2479085,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140520.918, "dur": 56.297,
    "args": {
      "External id": 2479086,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140613.600, "dur": 123.597,
    "args": {
      "External id": 2479087,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140626.967, "dur": 4.592,
    "args": {
      "External id": 2479088,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140636.788, "dur": 10.957,
    "args": {
      "External id": 2479089,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140641.159, "dur": 6.173,
    "args": {
      "External id": 2479090,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140645.047, "dur": 0.861,
    "args": {
      "External id": 2479091,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140655.139, "dur": 37.898,
    "args": {
      "External id": 2479092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140657.519, "dur": 2.459,
    "args": {
      "External id": 2479093,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140661.991, "dur": 0.671,
    "args": {
      "External id": 2479094,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140664.686, "dur": 0.562,
    "args": {
      "External id": 2479095,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140669.661, "dur": 2.455,
    "args": {
      "External id": 2479096,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140673.424, "dur": 0.662,
    "args": {
      "External id": 2479097,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140675.630, "dur": 0.612,
    "args": {
      "External id": 2479098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140680.011, "dur": 0.446,
    "args": {
      "External id": 2479099,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140682.300, "dur": 0.610,
    "args": {
      "External id": 2479100,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140684.367, "dur": 2.701,
    "args": {
      "External id": 2479101,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140705.180, "dur": 24.037,
    "args": {
      "External id": 2479102,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140790.310, "dur": 424.236,
    "args": {
      "External id": 2479103,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140823.221, "dur": 385.392,
    "args": {
      "External id": 2479104,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6807, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656140861.159, "dur": 341.252,
    "args": {
      "External id": 2479105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141244.518, "dur": 2.128,
    "args": {
      "External id": 2479106,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6809, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141343.325, "dur": 16803.881,
    "args": {
      "External id": 2479107,"Record function id": 0, "Ev Idx": 6810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141445.635, "dur": 6.875,
    "args": {
      "External id": 2479108,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141456.025, "dur": 1.223,
    "args": {
      "External id": 2479109,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141459.303, "dur": 3.715,
    "args": {
      "External id": 2479110,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141464.630, "dur": 1.000,
    "args": {
      "External id": 2479111,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141467.685, "dur": 1.242,
    "args": {
      "External id": 2479112,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141470.418, "dur": 1.170,
    "args": {
      "External id": 2479113,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141475.800, "dur": 1.460,
    "args": {
      "External id": 2479114,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141478.748, "dur": 2.475,
    "args": {
      "External id": 2479115,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141482.981, "dur": 0.745,
    "args": {
      "External id": 2479116,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141485.191, "dur": 0.808,
    "args": {
      "External id": 2479117,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141506.423, "dur": 16601.114,
    "args": {
      "External id": 2479118,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141521.599, "dur": 16578.816,
    "args": {
      "External id": 2479119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141557.743, "dur": 15.110,
    "args": {
      "External id": 2479120,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141576.222, "dur": 16488.936,
    "args": {
      "External id": 2479121,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141578.613, "dur": 16486.029,
    "args": {
      "External id": 2479122,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141585.311, "dur": 7.101,
    "args": {
      "External id": 2479123,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656141594.182, "dur": 16467.555,
    "args": {
      "External id": 2479124,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158288.968, "dur": 34.224,
    "args": {
      "External id": 2479125,"Sequence number": 24740441, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6828
    }
  },
  {
    "ph": "s", "id": 207, "pid": 1336760, "tid": 1336760, "ts": 1514656158288.968,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158311.336, "dur": 7.289,
    "args": {
      "External id": 2479126,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158314.169, "dur": 4.282,
    "args": {
      "External id": 2479127,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158388.001, "dur": 80.593,
    "args": {
      "External id": 2479128,"Record function id": 0, "Ev Idx": 6831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158470.555, "dur": 1107.397,
    "args": {
      "External id": 2479129,"Record function id": 0, "Ev Idx": 6832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158510.612, "dur": 1052.945,
    "args": {
      "External id": 2479130,"Sequence number": 24740442, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6833
    }
  },
  {
    "ph": "s", "id": 206, "pid": 1336760, "tid": 1336760, "ts": 1514656158510.612,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158592.043, "dur": 44.437,
    "args": {
      "External id": 2479131,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158650.092, "dur": 105.083,
    "args": {
      "External id": 2479132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158765.322, "dur": 40.626,
    "args": {
      "External id": 2479133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158814.155, "dur": 32.269,
    "args": {
      "External id": 2479134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158873.289, "dur": 28.014,
    "args": {
      "External id": 2479135,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158918.137, "dur": 15.904,
    "args": {
      "External id": 2479136,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656158953.134, "dur": 171.311,
    "args": {
      "External id": 2479137,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159037.375, "dur": 12.140,
    "args": {
      "External id": 2479138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159042.349, "dur": 6.111,
    "args": {
      "External id": 2479139,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159052.484, "dur": 6.230,
    "args": {
      "External id": 2479140,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159059.917, "dur": 0.852,
    "args": {
      "External id": 2479141,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159063.382, "dur": 6.814,
    "args": {
      "External id": 2479142,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159135.580, "dur": 51.845,
    "args": {
      "External id": 2479143,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159219.861, "dur": 32.132,
    "args": {
      "External id": 2479144,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159263.647, "dur": 41.676,
    "args": {
      "External id": 2479145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159314.049, "dur": 35.779,
    "args": {
      "External id": 2479146,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159371.428, "dur": 28.991,
    "args": {
      "External id": 2479147,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159406.179, "dur": 35.627,
    "args": {
      "External id": 2479148,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159460.950, "dur": 19.716,
    "args": {
      "External id": 2479149,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6852
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.10)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159642.830, "dur": 76.765,
    "args": {
      "External id": 2479150,"Record function id": 0, "Ev Idx": 6853
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159793.543, "dur": 48.170,
    "args": {
      "External id": 2479151,"Record function id": 0, "Ev Idx": 6854
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.11)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159850.175, "dur": 17997.578,
    "args": {
      "External id": 2479152,"Record function id": 0, "Ev Idx": 6855
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159858.655, "dur": 916.021,
    "args": {
      "External id": 2479153,"Record function id": 0, "Ev Idx": 6856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159944.377, "dur": 7.890,
    "args": {
      "External id": 2479154,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159965.417, "dur": 81.485,
    "args": {
      "External id": 2479155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159971.080, "dur": 2.295,
    "args": {
      "External id": 2479156,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159978.041, "dur": 0.394,
    "args": {
      "External id": 2479157,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656159980.111, "dur": 0.590,
    "args": {
      "External id": 2479158,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160019.807, "dur": 0.974,
    "args": {
      "External id": 2479159,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160025.081, "dur": 0.755,
    "args": {
      "External id": 2479160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160027.774, "dur": 0.873,
    "args": {
      "External id": 2479161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160030.298, "dur": 5.169,
    "args": {
      "External id": 2479162,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160037.344, "dur": 0.403,
    "args": {
      "External id": 2479163,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160038.968, "dur": 0.584,
    "args": {
      "External id": 2479164,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160063.173, "dur": 45.531,
    "args": {
      "External id": 2479165,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160143.795, "dur": 122.144,
    "args": {
      "External id": 2479166,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160158.736, "dur": 6.916,
    "args": {
      "External id": 2479167,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160170.921, "dur": 10.604,
    "args": {
      "External id": 2479168,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160175.394, "dur": 5.695,
    "args": {
      "External id": 2479169,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160179.227, "dur": 0.709,
    "args": {
      "External id": 2479170,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160188.970, "dur": 31.812,
    "args": {
      "External id": 2479171,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160191.385, "dur": 0.662,
    "args": {
      "External id": 2479172,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160193.926, "dur": 2.495,
    "args": {
      "External id": 2479173,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160198.189, "dur": 0.510,
    "args": {
      "External id": 2479174,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160200.465, "dur": 2.280,
    "args": {
      "External id": 2479175,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160206.683, "dur": 0.396,
    "args": {
      "External id": 2479176,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160208.739, "dur": 0.366,
    "args": {
      "External id": 2479177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160210.459, "dur": 0.424,
    "args": {
      "External id": 2479178,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160214.085, "dur": 0.349,
    "args": {
      "External id": 2479179,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160215.538, "dur": 0.545,
    "args": {
      "External id": 2479180,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160234.137, "dur": 23.598,
    "args": {
      "External id": 2479181,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160318.718, "dur": 359.969,
    "args": {
      "External id": 2479182,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160350.070, "dur": 323.804,
    "args": {
      "External id": 2479183,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6886, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160362.087, "dur": 304.271,
    "args": {
      "External id": 2479184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160705.199, "dur": 2.400,
    "args": {
      "External id": 2479185,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6888, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160794.069, "dur": 16858.220,
    "args": {
      "External id": 2479186,"Record function id": 0, "Ev Idx": 6889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160897.449, "dur": 6.455,
    "args": {
      "External id": 2479187,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160906.997, "dur": 1.545,
    "args": {
      "External id": 2479188,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160910.255, "dur": 3.163,
    "args": {
      "External id": 2479189,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160915.283, "dur": 1.168,
    "args": {
      "External id": 2479190,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160917.818, "dur": 1.294,
    "args": {
      "External id": 2479191,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160920.378, "dur": 1.351,
    "args": {
      "External id": 2479192,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160925.297, "dur": 0.970,
    "args": {
      "External id": 2479193,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160927.811, "dur": 2.272,
    "args": {
      "External id": 2479194,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160931.840, "dur": 0.935,
    "args": {
      "External id": 2479195,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160934.130, "dur": 0.993,
    "args": {
      "External id": 2479196,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160955.734, "dur": 16659.747,
    "args": {
      "External id": 2479197,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656160971.556, "dur": 16637.798,
    "args": {
      "External id": 2479198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656161028.525, "dur": 16.133,
    "args": {
      "External id": 2479199,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656161047.974, "dur": 16527.869,
    "args": {
      "External id": 2479200,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656161050.657, "dur": 16524.705,
    "args": {
      "External id": 2479201,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656161056.726, "dur": 7.930,
    "args": {
      "External id": 2479202,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656161066.292, "dur": 16506.380,
    "args": {
      "External id": 2479203,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656177787.911, "dur": 36.227,
    "args": {
      "External id": 2479204,"Sequence number": 24740443, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6907
    }
  },
  {
    "ph": "s", "id": 205, "pid": 1336760, "tid": 1336760, "ts": 1514656177787.911,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656177811.600, "dur": 7.716,
    "args": {
      "External id": 2479205,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656177814.684, "dur": 4.476,
    "args": {
      "External id": 2479206,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656177886.655, "dur": 84.077,
    "args": {
      "External id": 2479207,"Record function id": 0, "Ev Idx": 6910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656177972.664, "dur": 1132.956,
    "args": {
      "External id": 2479208,"Record function id": 0, "Ev Idx": 6911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178056.718, "dur": 1035.628,
    "args": {
      "External id": 2479209,"Sequence number": 24740444, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6912
    }
  },
  {
    "ph": "s", "id": 204, "pid": 1336760, "tid": 1336760, "ts": 1514656178056.718,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178123.359, "dur": 43.505,
    "args": {
      "External id": 2479210,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178180.382, "dur": 105.586,
    "args": {
      "External id": 2479211,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178296.204, "dur": 38.914,
    "args": {
      "External id": 2479212,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178343.149, "dur": 31.209,
    "args": {
      "External id": 2479213,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178400.115, "dur": 27.643,
    "args": {
      "External id": 2479214,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178444.977, "dur": 16.557,
    "args": {
      "External id": 2479215,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178480.407, "dur": 148.823,
    "args": {
      "External id": 2479216,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178542.437, "dur": 13.166,
    "args": {
      "External id": 2479217,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178548.431, "dur": 6.277,
    "args": {
      "External id": 2479218,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178558.699, "dur": 6.399,
    "args": {
      "External id": 2479219,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178566.660, "dur": 1.141,
    "args": {
      "External id": 2479220,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178570.411, "dur": 6.496,
    "args": {
      "External id": 2479221,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178640.382, "dur": 51.657,
    "args": {
      "External id": 2479222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178723.716, "dur": 32.871,
    "args": {
      "External id": 2479223,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178766.199, "dur": 42.543,
    "args": {
      "External id": 2479224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178817.981, "dur": 35.768,
    "args": {
      "External id": 2479225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178874.130, "dur": 31.624,
    "args": {
      "External id": 2479226,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178911.654, "dur": 34.506,
    "args": {
      "External id": 2479227,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656178967.587, "dur": 51.769,
    "args": {
      "External id": 2479228,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6931
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.11)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179168.874, "dur": 78.992,
    "args": {
      "External id": 2479229,"Record function id": 0, "Ev Idx": 6932
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179322.150, "dur": 45.347,
    "args": {
      "External id": 2479230,"Record function id": 0, "Ev Idx": 6933
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.12)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179376.683, "dur": 18072.064,
    "args": {
      "External id": 2479231,"Record function id": 0, "Ev Idx": 6934
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179385.078, "dur": 928.263,
    "args": {
      "External id": 2479232,"Record function id": 0, "Ev Idx": 6935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179472.700, "dur": 7.527,
    "args": {
      "External id": 2479233,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179496.749, "dur": 57.037,
    "args": {
      "External id": 2479234,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179502.350, "dur": 2.527,
    "args": {
      "External id": 2479235,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179509.474, "dur": 0.744,
    "args": {
      "External id": 2479236,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179511.772, "dur": 0.410,
    "args": {
      "External id": 2479237,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179514.190, "dur": 2.381,
    "args": {
      "External id": 2479238,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179518.327, "dur": 0.338,
    "args": {
      "External id": 2479239,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179520.553, "dur": 0.372,
    "args": {
      "External id": 2479240,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179524.774, "dur": 16.725,
    "args": {
      "External id": 2479241,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179544.509, "dur": 0.595,
    "args": {
      "External id": 2479242,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179546.479, "dur": 0.491,
    "args": {
      "External id": 2479243,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179567.809, "dur": 43.191,
    "args": {
      "External id": 2479244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179646.737, "dur": 120.000,
    "args": {
      "External id": 2479245,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179660.251, "dur": 4.703,
    "args": {
      "External id": 2479246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179670.248, "dur": 12.642,
    "args": {
      "External id": 2479247,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179674.617, "dur": 7.814,
    "args": {
      "External id": 2479248,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179678.285, "dur": 2.825,
    "args": {
      "External id": 2479249,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179690.634, "dur": 31.915,
    "args": {
      "External id": 2479250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179693.196, "dur": 0.700,
    "args": {
      "External id": 2479251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179695.320, "dur": 0.698,
    "args": {
      "External id": 2479252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179697.881, "dur": 0.350,
    "args": {
      "External id": 2479253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179701.914, "dur": 1.795,
    "args": {
      "External id": 2479254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179705.402, "dur": 0.367,
    "args": {
      "External id": 2479255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179707.014, "dur": 2.674,
    "args": {
      "External id": 2479256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179711.427, "dur": 0.372,
    "args": {
      "External id": 2479257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179713.456, "dur": 0.415,
    "args": {
      "External id": 2479258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179717.454, "dur": 0.429,
    "args": {
      "External id": 2479259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179734.064, "dur": 24.586,
    "args": {
      "External id": 2479260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179820.357, "dur": 392.329,
    "args": {
      "External id": 2479261,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179851.022, "dur": 356.360,
    "args": {
      "External id": 2479262,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6965, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656179863.855, "dur": 337.150,
    "args": {
      "External id": 2479263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180240.586, "dur": 2.405,
    "args": {
      "External id": 2479264,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6967, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180335.104, "dur": 16916.460,
    "args": {
      "External id": 2479265,"Record function id": 0, "Ev Idx": 6968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180433.859, "dur": 6.785,
    "args": {
      "External id": 2479266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180444.502, "dur": 1.138,
    "args": {
      "External id": 2479267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180447.224, "dur": 2.834,
    "args": {
      "External id": 2479268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180451.710, "dur": 1.003,
    "args": {
      "External id": 2479269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180454.456, "dur": 1.060,
    "args": {
      "External id": 2479270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180456.807, "dur": 0.898,
    "args": {
      "External id": 2479271,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180461.573, "dur": 1.167,
    "args": {
      "External id": 2479272,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180464.211, "dur": 2.214,
    "args": {
      "External id": 2479273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180468.152, "dur": 0.882,
    "args": {
      "External id": 2479274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180470.912, "dur": 0.722,
    "args": {
      "External id": 2479275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180492.689, "dur": 16721.639,
    "args": {
      "External id": 2479276,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180507.994, "dur": 16699.370,
    "args": {
      "External id": 2479277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180549.420, "dur": 14.438,
    "args": {
      "External id": 2479278,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180566.911, "dur": 16605.881,
    "args": {
      "External id": 2479279,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180569.178, "dur": 16603.070,
    "args": {
      "External id": 2479280,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180575.773, "dur": 6.525,
    "args": {
      "External id": 2479281,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656180583.999, "dur": 16585.473,
    "args": {
      "External id": 2479282,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197388.693, "dur": 35.583,
    "args": {
      "External id": 2479283,"Sequence number": 24740445, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6986
    }
  },
  {
    "ph": "s", "id": 203, "pid": 1336760, "tid": 1336760, "ts": 1514656197388.693,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197411.423, "dur": 7.989,
    "args": {
      "External id": 2479284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197414.900, "dur": 4.300,
    "args": {
      "External id": 2479285,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197487.790, "dur": 99.461,
    "args": {
      "External id": 2479286,"Record function id": 0, "Ev Idx": 6989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197590.391, "dur": 1102.404,
    "args": {
      "External id": 2479287,"Record function id": 0, "Ev Idx": 6990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197633.754, "dur": 1044.922,
    "args": {
      "External id": 2479288,"Sequence number": 24740446, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6991
    }
  },
  {
    "ph": "s", "id": 202, "pid": 1336760, "tid": 1336760, "ts": 1514656197633.754,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197700.550, "dur": 46.464,
    "args": {
      "External id": 2479289,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197759.039, "dur": 105.674,
    "args": {
      "External id": 2479290,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197873.958, "dur": 39.616,
    "args": {
      "External id": 2479291,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197921.252, "dur": 31.605,
    "args": {
      "External id": 2479292,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656197978.415, "dur": 65.060,
    "args": {
      "External id": 2479293,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198065.162, "dur": 17.329,
    "args": {
      "External id": 2479294,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198101.503, "dur": 131.697,
    "args": {
      "External id": 2479295,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198152.344, "dur": 11.999,
    "args": {
      "External id": 2479296,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198157.699, "dur": 5.846,
    "args": {
      "External id": 2479297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198167.333, "dur": 5.554,
    "args": {
      "External id": 2479298,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198174.075, "dur": 1.108,
    "args": {
      "External id": 2479299,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198177.520, "dur": 4.908,
    "args": {
      "External id": 2479300,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198244.242, "dur": 52.597,
    "args": {
      "External id": 2479301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198325.957, "dur": 29.680,
    "args": {
      "External id": 2479302,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198366.074, "dur": 41.556,
    "args": {
      "External id": 2479303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198415.971, "dur": 35.471,
    "args": {
      "External id": 2479304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198478.595, "dur": 29.957,
    "args": {
      "External id": 2479305,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198516.564, "dur": 52.080,
    "args": {
      "External id": 2479306,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198591.653, "dur": 20.109,
    "args": {
      "External id": 2479307,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7010
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.12)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198754.226, "dur": 75.492,
    "args": {
      "External id": 2479308,"Record function id": 0, "Ev Idx": 7011
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198905.358, "dur": 47.973,
    "args": {
      "External id": 2479309,"Record function id": 0, "Ev Idx": 7012
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.13)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198962.424, "dur": 18015.330,
    "args": {
      "External id": 2479310,"Record function id": 0, "Ev Idx": 7013
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656198971.264, "dur": 908.595,
    "args": {
      "External id": 2479311,"Record function id": 0, "Ev Idx": 7014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199094.025, "dur": 9.050,
    "args": {
      "External id": 2479312,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199117.219, "dur": 40.350,
    "args": {
      "External id": 2479313,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199122.812, "dur": 2.382,
    "args": {
      "External id": 2479314,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199129.874, "dur": 0.457,
    "args": {
      "External id": 2479315,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199131.894, "dur": 0.648,
    "args": {
      "External id": 2479316,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199134.046, "dur": 0.528,
    "args": {
      "External id": 2479317,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199138.765, "dur": 0.443,
    "args": {
      "External id": 2479318,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199140.603, "dur": 0.582,
    "args": {
      "External id": 2479319,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199142.599, "dur": 4.374,
    "args": {
      "External id": 2479320,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199148.542, "dur": 0.192,
    "args": {
      "External id": 2479321,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199150.231, "dur": 0.369,
    "args": {
      "External id": 2479322,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199170.172, "dur": 42.681,
    "args": {
      "External id": 2479323,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199246.369, "dur": 119.394,
    "args": {
      "External id": 2479324,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199260.243, "dur": 3.417,
    "args": {
      "External id": 2479325,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199268.643, "dur": 10.395,
    "args": {
      "External id": 2479326,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199272.903, "dur": 5.724,
    "args": {
      "External id": 2479327,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199276.604, "dur": 0.680,
    "args": {
      "External id": 2479328,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199286.353, "dur": 33.276,
    "args": {
      "External id": 2479329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199288.742, "dur": 2.656,
    "args": {
      "External id": 2479330,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199292.867, "dur": 0.710,
    "args": {
      "External id": 2479331,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199295.172, "dur": 0.616,
    "args": {
      "External id": 2479332,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199299.474, "dur": 1.774,
    "args": {
      "External id": 2479333,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199302.368, "dur": 0.398,
    "args": {
      "External id": 2479334,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199304.377, "dur": 0.475,
    "args": {
      "External id": 2479335,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199308.127, "dur": 0.180,
    "args": {
      "External id": 2479336,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199309.714, "dur": 0.530,
    "args": {
      "External id": 2479337,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199312.151, "dur": 2.453,
    "args": {
      "External id": 2479338,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199333.057, "dur": 24.754,
    "args": {
      "External id": 2479339,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199419.588, "dur": 364.522,
    "args": {
      "External id": 2479340,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199452.947, "dur": 326.258,
    "args": {
      "External id": 2479341,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7044, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199463.711, "dur": 309.064,
    "args": {
      "External id": 2479342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199810.815, "dur": 2.294,
    "args": {
      "External id": 2479343,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7046, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656199900.696, "dur": 16878.740,
    "args": {
      "External id": 2479344,"Record function id": 0, "Ev Idx": 7047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200041.195, "dur": 6.716,
    "args": {
      "External id": 2479345,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200051.718, "dur": 1.395,
    "args": {
      "External id": 2479346,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200055.252, "dur": 2.816,
    "args": {
      "External id": 2479347,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200059.657, "dur": 1.087,
    "args": {
      "External id": 2479348,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200062.190, "dur": 1.127,
    "args": {
      "External id": 2479349,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200064.530, "dur": 0.700,
    "args": {
      "External id": 2479350,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200068.943, "dur": 0.907,
    "args": {
      "External id": 2479351,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200071.386, "dur": 1.911,
    "args": {
      "External id": 2479352,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200075.153, "dur": 0.917,
    "args": {
      "External id": 2479353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200077.668, "dur": 1.026,
    "args": {
      "External id": 2479354,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200100.362, "dur": 16640.897,
    "args": {
      "External id": 2479355,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200116.548, "dur": 16618.318,
    "args": {
      "External id": 2479356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200142.493, "dur": 14.446,
    "args": {
      "External id": 2479357,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200160.192, "dur": 16540.615,
    "args": {
      "External id": 2479358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200162.638, "dur": 16537.706,
    "args": {
      "External id": 2479359,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200168.362, "dur": 5.137,
    "args": {
      "External id": 2479360,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656200175.109, "dur": 16522.362,
    "args": {
      "External id": 2479361,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656216919.881, "dur": 33.530,
    "args": {
      "External id": 2479362,"Sequence number": 24740447, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7065
    }
  },
  {
    "ph": "s", "id": 201, "pid": 1336760, "tid": 1336760, "ts": 1514656216919.881,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656216941.531, "dur": 7.410,
    "args": {
      "External id": 2479363,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656216944.676, "dur": 4.080,
    "args": {
      "External id": 2479364,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217046.089, "dur": 81.569,
    "args": {
      "External id": 2479365,"Record function id": 0, "Ev Idx": 7068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217129.733, "dur": 1100.227,
    "args": {
      "External id": 2479366,"Record function id": 0, "Ev Idx": 7069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217170.514, "dur": 1046.688,
    "args": {
      "External id": 2479367,"Sequence number": 24740448, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7070
    }
  },
  {
    "ph": "s", "id": 200, "pid": 1336760, "tid": 1336760, "ts": 1514656217170.514,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217240.270, "dur": 46.024,
    "args": {
      "External id": 2479368,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217298.193, "dur": 103.365,
    "args": {
      "External id": 2479369,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217411.421, "dur": 39.045,
    "args": {
      "External id": 2479370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217458.515, "dur": 31.587,
    "args": {
      "External id": 2479371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217516.335, "dur": 42.223,
    "args": {
      "External id": 2479372,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217579.951, "dur": 20.451,
    "args": {
      "External id": 2479373,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217618.600, "dur": 130.346,
    "args": {
      "External id": 2479374,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217667.564, "dur": 12.245,
    "args": {
      "External id": 2479375,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217672.981, "dur": 6.025,
    "args": {
      "External id": 2479376,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217682.947, "dur": 5.739,
    "args": {
      "External id": 2479377,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217690.040, "dur": 1.313,
    "args": {
      "External id": 2479378,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217693.788, "dur": 4.388,
    "args": {
      "External id": 2479379,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217759.422, "dur": 50.444,
    "args": {
      "External id": 2479380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217841.514, "dur": 28.983,
    "args": {
      "External id": 2479381,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217879.450, "dur": 42.839,
    "args": {
      "External id": 2479382,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656217930.814, "dur": 35.051,
    "args": {
      "External id": 2479383,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218024.389, "dur": 33.255,
    "args": {
      "External id": 2479384,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218064.648, "dur": 39.795,
    "args": {
      "External id": 2479385,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218126.324, "dur": 19.150,
    "args": {
      "External id": 2479386,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.13)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218292.245, "dur": 79.263,
    "args": {
      "External id": 2479387,"Record function id": 0, "Ev Idx": 7090
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218444.020, "dur": 46.354,
    "args": {
      "External id": 2479388,"Record function id": 0, "Ev Idx": 7091
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.14)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218499.860, "dur": 18178.539,
    "args": {
      "External id": 2479389,"Record function id": 0, "Ev Idx": 7092
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218509.266, "dur": 939.085,
    "args": {
      "External id": 2479390,"Record function id": 0, "Ev Idx": 7093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218609.345, "dur": 8.718,
    "args": {
      "External id": 2479391,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218632.054, "dur": 46.451,
    "args": {
      "External id": 2479392,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218641.278, "dur": 2.248,
    "args": {
      "External id": 2479393,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218647.912, "dur": 0.560,
    "args": {
      "External id": 2479394,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218649.925, "dur": 0.353,
    "args": {
      "External id": 2479395,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218651.803, "dur": 2.692,
    "args": {
      "External id": 2479396,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218656.303, "dur": 0.393,
    "args": {
      "External id": 2479397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218660.105, "dur": 0.329,
    "args": {
      "External id": 2479398,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218663.729, "dur": 1.696,
    "args": {
      "External id": 2479399,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218666.824, "dur": 0.301,
    "args": {
      "External id": 2479400,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218668.544, "dur": 2.744,
    "args": {
      "External id": 2479401,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218691.345, "dur": 40.800,
    "args": {
      "External id": 2479402,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218765.242, "dur": 111.774,
    "args": {
      "External id": 2479403,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218775.752, "dur": 3.479,
    "args": {
      "External id": 2479404,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218784.452, "dur": 9.858,
    "args": {
      "External id": 2479405,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218788.835, "dur": 5.028,
    "args": {
      "External id": 2479406,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218792.122, "dur": 0.543,
    "args": {
      "External id": 2479407,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218801.928, "dur": 32.761,
    "args": {
      "External id": 2479408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218804.231, "dur": 0.482,
    "args": {
      "External id": 2479409,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218806.451, "dur": 0.554,
    "args": {
      "External id": 2479410,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218808.309, "dur": 2.423,
    "args": {
      "External id": 2479411,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218812.435, "dur": 2.423,
    "args": {
      "External id": 2479412,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218816.494, "dur": 0.345,
    "args": {
      "External id": 2479413,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218821.905, "dur": 0.200,
    "args": {
      "External id": 2479414,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218823.726, "dur": 0.358,
    "args": {
      "External id": 2479415,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218825.742, "dur": 0.306,
    "args": {
      "External id": 2479416,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218829.710, "dur": 0.367,
    "args": {
      "External id": 2479417,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218845.813, "dur": 23.430,
    "args": {
      "External id": 2479418,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218930.530, "dur": 416.355,
    "args": {
      "External id": 2479419,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218963.959, "dur": 377.407,
    "args": {
      "External id": 2479420,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7123, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656218974.448, "dur": 361.112,
    "args": {
      "External id": 2479421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219374.350, "dur": 2.318,
    "args": {
      "External id": 2479422,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7125, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219470.065, "dur": 16992.699,
    "args": {
      "External id": 2479423,"Record function id": 0, "Ev Idx": 7126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219594.116, "dur": 7.005,
    "args": {
      "External id": 2479424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219605.010, "dur": 1.155,
    "args": {
      "External id": 2479425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219607.759, "dur": 3.106,
    "args": {
      "External id": 2479426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219612.512, "dur": 1.159,
    "args": {
      "External id": 2479427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219615.064, "dur": 1.054,
    "args": {
      "External id": 2479428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219617.710, "dur": 1.059,
    "args": {
      "External id": 2479429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219622.363, "dur": 1.033,
    "args": {
      "External id": 2479430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219625.134, "dur": 1.943,
    "args": {
      "External id": 2479431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219628.754, "dur": 0.875,
    "args": {
      "External id": 2479432,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219631.458, "dur": 1.363,
    "args": {
      "External id": 2479433,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219654.328, "dur": 16769.125,
    "args": {
      "External id": 2479434,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219671.238, "dur": 16745.475,
    "args": {
      "External id": 2479435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219696.416, "dur": 14.677,
    "args": {
      "External id": 2479436,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219714.067, "dur": 16669.124,
    "args": {
      "External id": 2479437,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219716.560, "dur": 16666.099,
    "args": {
      "External id": 2479438,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219722.890, "dur": 6.064,
    "args": {
      "External id": 2479439,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656219731.071, "dur": 16648.749,
    "args": {
      "External id": 2479440,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236615.721, "dur": 36.390,
    "args": {
      "External id": 2479441,"Sequence number": 24740449, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7144
    }
  },
  {
    "ph": "s", "id": 199, "pid": 1336760, "tid": 1336760, "ts": 1514656236615.721,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236639.826, "dur": 7.710,
    "args": {
      "External id": 2479442,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236642.997, "dur": 4.186,
    "args": {
      "External id": 2479443,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236720.735, "dur": 80.058,
    "args": {
      "External id": 2479444,"Record function id": 0, "Ev Idx": 7147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236802.051, "dur": 1095.950,
    "args": {
      "External id": 2479445,"Record function id": 0, "Ev Idx": 7148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236840.531, "dur": 1043.609,
    "args": {
      "External id": 2479446,"Sequence number": 24740450, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7149
    }
  },
  {
    "ph": "s", "id": 198, "pid": 1336760, "tid": 1336760, "ts": 1514656236840.531,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236905.610, "dur": 42.960,
    "args": {
      "External id": 2479447,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656236960.562, "dur": 132.501,
    "args": {
      "External id": 2479448,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237107.863, "dur": 41.186,
    "args": {
      "External id": 2479449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237158.828, "dur": 31.398,
    "args": {
      "External id": 2479450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237217.228, "dur": 28.839,
    "args": {
      "External id": 2479451,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237267.822, "dur": 16.250,
    "args": {
      "External id": 2479452,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237302.360, "dur": 137.694,
    "args": {
      "External id": 2479453,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237357.195, "dur": 13.585,
    "args": {
      "External id": 2479454,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237363.841, "dur": 6.234,
    "args": {
      "External id": 2479455,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237373.691, "dur": 6.122,
    "args": {
      "External id": 2479456,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237381.182, "dur": 1.515,
    "args": {
      "External id": 2479457,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237385.324, "dur": 5.106,
    "args": {
      "External id": 2479458,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237450.432, "dur": 47.342,
    "args": {
      "External id": 2479459,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237544.799, "dur": 33.705,
    "args": {
      "External id": 2479460,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237589.946, "dur": 45.390,
    "args": {
      "External id": 2479461,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237643.732, "dur": 36.065,
    "args": {
      "External id": 2479462,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237703.539, "dur": 29.868,
    "args": {
      "External id": 2479463,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237738.787, "dur": 34.337,
    "args": {
      "External id": 2479464,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237795.414, "dur": 19.405,
    "args": {
      "External id": 2479465,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7168
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.14)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656237962.234, "dur": 115.131,
    "args": {
      "External id": 2479466,"Record function id": 0, "Ev Idx": 7169
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238156.372, "dur": 47.743,
    "args": {
      "External id": 2479467,"Record function id": 0, "Ev Idx": 7170
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.15)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238213.507, "dur": 18112.087,
    "args": {
      "External id": 2479468,"Record function id": 0, "Ev Idx": 7171
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238223.246, "dur": 848.650,
    "args": {
      "External id": 2479469,"Record function id": 0, "Ev Idx": 7172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238307.222, "dur": 9.305,
    "args": {
      "External id": 2479470,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238330.188, "dur": 40.738,
    "args": {
      "External id": 2479471,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238335.534, "dur": 2.500,
    "args": {
      "External id": 2479472,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238343.126, "dur": 0.420,
    "args": {
      "External id": 2479473,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238345.448, "dur": 0.352,
    "args": {
      "External id": 2479474,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238347.394, "dur": 0.424,
    "args": {
      "External id": 2479475,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238351.773, "dur": 0.786,
    "args": {
      "External id": 2479476,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238354.040, "dur": 0.407,
    "args": {
      "External id": 2479477,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238356.236, "dur": 4.059,
    "args": {
      "External id": 2479478,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238361.998, "dur": 0.324,
    "args": {
      "External id": 2479479,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238363.769, "dur": 0.457,
    "args": {
      "External id": 2479480,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238383.348, "dur": 43.922,
    "args": {
      "External id": 2479481,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238459.440, "dur": 147.390,
    "args": {
      "External id": 2479482,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238469.973, "dur": 3.893,
    "args": {
      "External id": 2479483,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238478.944, "dur": 9.749,
    "args": {
      "External id": 2479484,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238483.211, "dur": 5.082,
    "args": {
      "External id": 2479485,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238486.487, "dur": 0.653,
    "args": {
      "External id": 2479486,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238495.996, "dur": 50.791,
    "args": {
      "External id": 2479487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238498.367, "dur": 3.118,
    "args": {
      "External id": 2479488,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238503.007, "dur": 0.494,
    "args": {
      "External id": 2479489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238505.166, "dur": 0.439,
    "args": {
      "External id": 2479490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238509.518, "dur": 2.474,
    "args": {
      "External id": 2479491,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238514.642, "dur": 0.262,
    "args": {
      "External id": 2479492,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238516.448, "dur": 0.311,
    "args": {
      "External id": 2479493,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238520.289, "dur": 0.403,
    "args": {
      "External id": 2479494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238522.513, "dur": 0.253,
    "args": {
      "External id": 2479495,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238524.124, "dur": 16.727,
    "args": {
      "External id": 2479496,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238570.388, "dur": 27.984,
    "args": {
      "External id": 2479497,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238660.861, "dur": 283.846,
    "args": {
      "External id": 2479498,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238695.398, "dur": 244.815,
    "args": {
      "External id": 2479499,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7202, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238708.031, "dur": 227.193,
    "args": {
      "External id": 2479500,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656238968.619, "dur": 2.256,
    "args": {
      "External id": 2479501,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7204, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239093.468, "dur": 17037.382,
    "args": {
      "External id": 2479502,"Record function id": 0, "Ev Idx": 7205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239195.720, "dur": 6.397,
    "args": {
      "External id": 2479503,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239205.850, "dur": 1.213,
    "args": {
      "External id": 2479504,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239208.749, "dur": 2.847,
    "args": {
      "External id": 2479505,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239213.157, "dur": 1.523,
    "args": {
      "External id": 2479506,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239215.996, "dur": 1.181,
    "args": {
      "External id": 2479507,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239218.645, "dur": 1.184,
    "args": {
      "External id": 2479508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239223.023, "dur": 0.935,
    "args": {
      "External id": 2479509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239225.619, "dur": 2.276,
    "args": {
      "External id": 2479510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239229.451, "dur": 0.777,
    "args": {
      "External id": 2479511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239231.707, "dur": 0.735,
    "args": {
      "External id": 2479512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239252.504, "dur": 16841.917,
    "args": {
      "External id": 2479513,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239267.482, "dur": 16820.483,
    "args": {
      "External id": 2479514,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239290.682, "dur": 14.485,
    "args": {
      "External id": 2479515,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239308.021, "dur": 16745.554,
    "args": {
      "External id": 2479516,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239310.711, "dur": 16742.296,
    "args": {
      "External id": 2479517,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239316.872, "dur": 5.239,
    "args": {
      "External id": 2479518,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656239323.713, "dur": 16726.503,
    "args": {
      "External id": 2479519,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256265.598, "dur": 37.091,
    "args": {
      "External id": 2479520,"Sequence number": 24740451, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7223
    }
  },
  {
    "ph": "s", "id": 197, "pid": 1336760, "tid": 1336760, "ts": 1514656256265.598,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256290.160, "dur": 7.759,
    "args": {
      "External id": 2479521,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256293.376, "dur": 4.366,
    "args": {
      "External id": 2479522,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256365.510, "dur": 81.053,
    "args": {
      "External id": 2479523,"Record function id": 0, "Ev Idx": 7226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256448.398, "dur": 1138.885,
    "args": {
      "External id": 2479524,"Record function id": 0, "Ev Idx": 7227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256490.605, "dur": 1081.778,
    "args": {
      "External id": 2479525,"Sequence number": 24740452, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7228
    }
  },
  {
    "ph": "s", "id": 196, "pid": 1336760, "tid": 1336760, "ts": 1514656256490.605,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256573.132, "dur": 44.264,
    "args": {
      "External id": 2479526,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256631.444, "dur": 107.203,
    "args": {
      "External id": 2479527,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256749.031, "dur": 39.677,
    "args": {
      "External id": 2479528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256796.560, "dur": 32.190,
    "args": {
      "External id": 2479529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256853.981, "dur": 28.916,
    "args": {
      "External id": 2479530,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256901.337, "dur": 18.258,
    "args": {
      "External id": 2479531,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656256936.944, "dur": 177.486,
    "args": {
      "External id": 2479532,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257026.726, "dur": 12.929,
    "args": {
      "External id": 2479533,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257031.853, "dur": 6.789,
    "args": {
      "External id": 2479534,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257042.755, "dur": 5.949,
    "args": {
      "External id": 2479535,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257050.023, "dur": 1.341,
    "args": {
      "External id": 2479536,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257053.942, "dur": 5.075,
    "args": {
      "External id": 2479537,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257125.277, "dur": 54.700,
    "args": {
      "External id": 2479538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257213.912, "dur": 32.059,
    "args": {
      "External id": 2479539,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257255.614, "dur": 43.783,
    "args": {
      "External id": 2479540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257308.971, "dur": 35.098,
    "args": {
      "External id": 2479541,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257368.630, "dur": 30.831,
    "args": {
      "External id": 2479542,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257404.889, "dur": 33.522,
    "args": {
      "External id": 2479543,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257459.460, "dur": 19.251,
    "args": {
      "External id": 2479544,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7247
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.15)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257654.425, "dur": 82.527,
    "args": {
      "External id": 2479545,"Record function id": 0, "Ev Idx": 7248
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257811.981, "dur": 45.620,
    "args": {
      "External id": 2479546,"Record function id": 0, "Ev Idx": 7249
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.16)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257867.311, "dur": 17988.217,
    "args": {
      "External id": 2479547,"Record function id": 0, "Ev Idx": 7250
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257877.181, "dur": 913.651,
    "args": {
      "External id": 2479548,"Record function id": 0, "Ev Idx": 7251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656257959.486, "dur": 8.261,
    "args": {
      "External id": 2479549,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258016.733, "dur": 41.110,
    "args": {
      "External id": 2479550,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258023.305, "dur": 2.411,
    "args": {
      "External id": 2479551,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258030.893, "dur": 0.260,
    "args": {
      "External id": 2479552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258032.906, "dur": 0.519,
    "args": {
      "External id": 2479553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258034.835, "dur": 0.840,
    "args": {
      "External id": 2479554,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258038.774, "dur": 0.697,
    "args": {
      "External id": 2479555,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258040.739, "dur": 0.632,
    "args": {
      "External id": 2479556,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258043.113, "dur": 4.327,
    "args": {
      "External id": 2479557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258048.680, "dur": 0.576,
    "args": {
      "External id": 2479558,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258050.802, "dur": 0.464,
    "args": {
      "External id": 2479559,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258074.996, "dur": 42.103,
    "args": {
      "External id": 2479560,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258151.524, "dur": 119.583,
    "args": {
      "External id": 2479561,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258163.186, "dur": 7.354,
    "args": {
      "External id": 2479562,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258175.919, "dur": 10.869,
    "args": {
      "External id": 2479563,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258180.547, "dur": 5.798,
    "args": {
      "External id": 2479564,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258184.323, "dur": 0.652,
    "args": {
      "External id": 2479565,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258194.608, "dur": 32.883,
    "args": {
      "External id": 2479566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258196.942, "dur": 0.488,
    "args": {
      "External id": 2479567,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258198.921, "dur": 2.775,
    "args": {
      "External id": 2479568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258203.232, "dur": 0.403,
    "args": {
      "External id": 2479569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258205.260, "dur": 2.252,
    "args": {
      "External id": 2479570,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258211.385, "dur": 0.351,
    "args": {
      "External id": 2479571,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258213.405, "dur": 0.511,
    "args": {
      "External id": 2479572,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258215.385, "dur": 0.392,
    "args": {
      "External id": 2479573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258219.470, "dur": 0.387,
    "args": {
      "External id": 2479574,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258221.214, "dur": 0.298,
    "args": {
      "External id": 2479575,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258239.371, "dur": 23.688,
    "args": {
      "External id": 2479576,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258326.666, "dur": 366.589,
    "args": {
      "External id": 2479577,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258361.313, "dur": 327.229,
    "args": {
      "External id": 2479578,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7281, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258372.226, "dur": 310.436,
    "args": {
      "External id": 2479579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258720.150, "dur": 2.470,
    "args": {
      "External id": 2479580,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7283, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258812.549, "dur": 16848.187,
    "args": {
      "External id": 2479581,"Record function id": 0, "Ev Idx": 7284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258914.668, "dur": 6.535,
    "args": {
      "External id": 2479582,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258924.446, "dur": 1.127,
    "args": {
      "External id": 2479583,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258927.325, "dur": 2.353,
    "args": {
      "External id": 2479584,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258931.254, "dur": 1.015,
    "args": {
      "External id": 2479585,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258933.843, "dur": 0.996,
    "args": {
      "External id": 2479586,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258936.107, "dur": 0.662,
    "args": {
      "External id": 2479587,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258940.396, "dur": 1.264,
    "args": {
      "External id": 2479588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258943.229, "dur": 2.247,
    "args": {
      "External id": 2479589,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258947.129, "dur": 1.062,
    "args": {
      "External id": 2479590,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258949.751, "dur": 1.172,
    "args": {
      "External id": 2479591,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656258971.531, "dur": 16651.663,
    "args": {
      "External id": 2479592,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259023.074, "dur": 16593.262,
    "args": {
      "External id": 2479593,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259050.273, "dur": 13.544,
    "args": {
      "External id": 2479594,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259066.913, "dur": 16515.989,
    "args": {
      "External id": 2479595,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259069.460, "dur": 16512.900,
    "args": {
      "External id": 2479596,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259075.997, "dur": 6.122,
    "args": {
      "External id": 2479597,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656259083.835, "dur": 16495.904,
    "args": {
      "External id": 2479598,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656275796.712, "dur": 35.225,
    "args": {
      "External id": 2479599,"Sequence number": 24740453, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7302
    }
  },
  {
    "ph": "s", "id": 195, "pid": 1336760, "tid": 1336760, "ts": 1514656275796.712,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656275819.415, "dur": 7.826,
    "args": {
      "External id": 2479600,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656275822.777, "dur": 4.270,
    "args": {
      "External id": 2479601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656275894.220, "dur": 78.614,
    "args": {
      "External id": 2479602,"Record function id": 0, "Ev Idx": 7305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656275974.572, "dur": 1191.050,
    "args": {
      "External id": 2479603,"Record function id": 0, "Ev Idx": 7306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276059.685, "dur": 1092.745,
    "args": {
      "External id": 2479604,"Sequence number": 24740454, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7307
    }
  },
  {
    "ph": "s", "id": 194, "pid": 1336760, "tid": 1336760, "ts": 1514656276059.685,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276127.837, "dur": 51.660,
    "args": {
      "External id": 2479605,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276192.403, "dur": 102.705,
    "args": {
      "External id": 2479606,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276305.457, "dur": 40.410,
    "args": {
      "External id": 2479607,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276354.754, "dur": 31.538,
    "args": {
      "External id": 2479608,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276418.840, "dur": 30.117,
    "args": {
      "External id": 2479609,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276465.616, "dur": 15.739,
    "args": {
      "External id": 2479610,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276497.922, "dur": 151.671,
    "args": {
      "External id": 2479611,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276563.617, "dur": 13.042,
    "args": {
      "External id": 2479612,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276569.093, "dur": 6.571,
    "args": {
      "External id": 2479613,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276579.724, "dur": 5.833,
    "args": {
      "External id": 2479614,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276586.768, "dur": 1.462,
    "args": {
      "External id": 2479615,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276590.755, "dur": 5.921,
    "args": {
      "External id": 2479616,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276686.829, "dur": 54.247,
    "args": {
      "External id": 2479617,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276776.916, "dur": 32.340,
    "args": {
      "External id": 2479618,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276819.154, "dur": 41.703,
    "args": {
      "External id": 2479619,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276870.016, "dur": 35.755,
    "args": {
      "External id": 2479620,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276927.836, "dur": 27.618,
    "args": {
      "External id": 2479621,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656276960.838, "dur": 69.217,
    "args": {
      "External id": 2479622,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277056.626, "dur": 24.559,
    "args": {
      "External id": 2479623,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7326
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.16)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277229.227, "dur": 79.408,
    "args": {
      "External id": 2479624,"Record function id": 0, "Ev Idx": 7327
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277383.715, "dur": 49.366,
    "args": {
      "External id": 2479625,"Record function id": 0, "Ev Idx": 7328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.17)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277442.375, "dur": 17938.864,
    "args": {
      "External id": 2479626,"Record function id": 0, "Ev Idx": 7329
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277450.348, "dur": 908.141,
    "args": {
      "External id": 2479627,"Record function id": 0, "Ev Idx": 7330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277550.696, "dur": 9.792,
    "args": {
      "External id": 2479628,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277575.305, "dur": 39.753,
    "args": {
      "External id": 2479629,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277580.637, "dur": 2.535,
    "args": {
      "External id": 2479630,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277587.833, "dur": 0.284,
    "args": {
      "External id": 2479631,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277589.830, "dur": 0.641,
    "args": {
      "External id": 2479632,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277591.901, "dur": 0.731,
    "args": {
      "External id": 2479633,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277596.294, "dur": 0.555,
    "args": {
      "External id": 2479634,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277598.522, "dur": 0.745,
    "args": {
      "External id": 2479635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277600.721, "dur": 3.295,
    "args": {
      "External id": 2479636,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277605.632, "dur": 0.390,
    "args": {
      "External id": 2479637,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277607.472, "dur": 0.374,
    "args": {
      "External id": 2479638,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277627.019, "dur": 41.618,
    "args": {
      "External id": 2479639,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277704.384, "dur": 118.155,
    "args": {
      "External id": 2479640,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277715.421, "dur": 3.940,
    "args": {
      "External id": 2479641,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277724.318, "dur": 10.569,
    "args": {
      "External id": 2479642,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277728.630, "dur": 5.850,
    "args": {
      "External id": 2479643,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277732.761, "dur": 0.606,
    "args": {
      "External id": 2479644,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277741.862, "dur": 35.686,
    "args": {
      "External id": 2479645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277744.281, "dur": 2.737,
    "args": {
      "External id": 2479646,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277748.926, "dur": 0.180,
    "args": {
      "External id": 2479647,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277750.792, "dur": 0.619,
    "args": {
      "External id": 2479648,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277755.253, "dur": 1.993,
    "args": {
      "External id": 2479649,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277758.702, "dur": 0.543,
    "args": {
      "External id": 2479650,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277760.584, "dur": 0.501,
    "args": {
      "External id": 2479651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277765.053, "dur": 0.542,
    "args": {
      "External id": 2479652,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277767.286, "dur": 0.556,
    "args": {
      "External id": 2479653,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277769.228, "dur": 3.021,
    "args": {
      "External id": 2479654,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277791.080, "dur": 23.432,
    "args": {
      "External id": 2479655,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277876.933, "dur": 385.728,
    "args": {
      "External id": 2479656,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277911.604, "dur": 345.840,
    "args": {
      "External id": 2479657,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7360, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656277922.391, "dur": 328.214,
    "args": {
      "External id": 2479658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278288.267, "dur": 2.694,
    "args": {
      "External id": 2479659,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7362, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278380.332, "dur": 16802.044,
    "args": {
      "External id": 2479660,"Record function id": 0, "Ev Idx": 7363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278481.799, "dur": 6.572,
    "args": {
      "External id": 2479661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278491.751, "dur": 1.433,
    "args": {
      "External id": 2479662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278494.760, "dur": 2.873,
    "args": {
      "External id": 2479663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278499.286, "dur": 0.816,
    "args": {
      "External id": 2479664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278501.268, "dur": 0.974,
    "args": {
      "External id": 2479665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278503.344, "dur": 0.708,
    "args": {
      "External id": 2479666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278507.755, "dur": 0.822,
    "args": {
      "External id": 2479667,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278510.101, "dur": 1.988,
    "args": {
      "External id": 2479668,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278513.633, "dur": 1.056,
    "args": {
      "External id": 2479669,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278516.029, "dur": 0.664,
    "args": {
      "External id": 2479670,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278554.822, "dur": 16587.998,
    "args": {
      "External id": 2479671,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278571.710, "dur": 16564.752,
    "args": {
      "External id": 2479672,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278602.374, "dur": 15.376,
    "args": {
      "External id": 2479673,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278620.719, "dur": 16480.927,
    "args": {
      "External id": 2479674,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278623.143, "dur": 16477.964,
    "args": {
      "External id": 2479675,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278629.294, "dur": 5.996,
    "args": {
      "External id": 2479676,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656278637.063, "dur": 16461.366,
    "args": {
      "External id": 2479677,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295323.365, "dur": 34.137,
    "args": {
      "External id": 2479678,"Sequence number": 24740455, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7381
    }
  },
  {
    "ph": "s", "id": 193, "pid": 1336760, "tid": 1336760, "ts": 1514656295323.365,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295345.347, "dur": 7.575,
    "args": {
      "External id": 2479679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295348.440, "dur": 4.332,
    "args": {
      "External id": 2479680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295421.918, "dur": 78.759,
    "args": {
      "External id": 2479681,"Record function id": 0, "Ev Idx": 7384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295502.569, "dur": 1110.252,
    "args": {
      "External id": 2479682,"Record function id": 0, "Ev Idx": 7385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295558.423, "dur": 1041.013,
    "args": {
      "External id": 2479683,"Sequence number": 24740456, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7386
    }
  },
  {
    "ph": "s", "id": 192, "pid": 1336760, "tid": 1336760, "ts": 1514656295558.423,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295629.499, "dur": 43.742,
    "args": {
      "External id": 2479684,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295686.459, "dur": 108.766,
    "args": {
      "External id": 2479685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295804.775, "dur": 39.140,
    "args": {
      "External id": 2479686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295852.742, "dur": 31.222,
    "args": {
      "External id": 2479687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295908.925, "dur": 28.383,
    "args": {
      "External id": 2479688,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656295954.758, "dur": 17.761,
    "args": {
      "External id": 2479689,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296023.827, "dur": 137.336,
    "args": {
      "External id": 2479690,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296074.778, "dur": 13.745,
    "args": {
      "External id": 2479691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296080.693, "dur": 6.872,
    "args": {
      "External id": 2479692,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296091.689, "dur": 5.925,
    "args": {
      "External id": 2479693,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296099.009, "dur": 1.357,
    "args": {
      "External id": 2479694,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296102.727, "dur": 4.996,
    "args": {
      "External id": 2479695,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296171.849, "dur": 52.130,
    "args": {
      "External id": 2479696,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296256.232, "dur": 29.745,
    "args": {
      "External id": 2479697,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296294.692, "dur": 43.716,
    "args": {
      "External id": 2479698,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296347.248, "dur": 35.545,
    "args": {
      "External id": 2479699,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296405.660, "dur": 29.437,
    "args": {
      "External id": 2479700,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296440.514, "dur": 34.613,
    "args": {
      "External id": 2479701,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296495.399, "dur": 18.394,
    "args": {
      "External id": 2479702,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7405
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.17)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296676.338, "dur": 74.267,
    "args": {
      "External id": 2479703,"Record function id": 0, "Ev Idx": 7406
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296822.973, "dur": 46.769,
    "args": {
      "External id": 2479704,"Record function id": 0, "Ev Idx": 7407
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.18)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296879.124, "dur": 18233.289,
    "args": {
      "External id": 2479705,"Record function id": 0, "Ev Idx": 7408
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296888.188, "dur": 927.694,
    "args": {
      "External id": 2479706,"Record function id": 0, "Ev Idx": 7409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656296971.491, "dur": 8.282,
    "args": {
      "External id": 2479707,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297030.984, "dur": 38.624,
    "args": {
      "External id": 2479708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297036.443, "dur": 2.716,
    "args": {
      "External id": 2479709,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297044.197, "dur": 0.753,
    "args": {
      "External id": 2479710,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297046.563, "dur": 0.487,
    "args": {
      "External id": 2479711,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297048.865, "dur": 0.608,
    "args": {
      "External id": 2479712,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297052.881, "dur": 0.209,
    "args": {
      "External id": 2479713,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297054.651, "dur": 0.384,
    "args": {
      "External id": 2479714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297056.793, "dur": 3.540,
    "args": {
      "External id": 2479715,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297061.559, "dur": 0.424,
    "args": {
      "External id": 2479716,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297063.391, "dur": 0.512,
    "args": {
      "External id": 2479717,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297083.008, "dur": 42.101,
    "args": {
      "External id": 2479718,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297159.566, "dur": 120.107,
    "args": {
      "External id": 2479719,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297173.607, "dur": 6.949,
    "args": {
      "External id": 2479720,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297185.942, "dur": 11.278,
    "args": {
      "External id": 2479721,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297190.385, "dur": 6.414,
    "args": {
      "External id": 2479722,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297194.556, "dur": 0.820,
    "args": {
      "External id": 2479723,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297203.979, "dur": 31.391,
    "args": {
      "External id": 2479724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297206.376, "dur": 0.520,
    "args": {
      "External id": 2479725,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297208.492, "dur": 2.784,
    "args": {
      "External id": 2479726,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297212.780, "dur": 0.506,
    "args": {
      "External id": 2479727,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297214.993, "dur": 2.291,
    "args": {
      "External id": 2479728,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297220.971, "dur": 0.145,
    "args": {
      "External id": 2479729,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297222.490, "dur": 0.399,
    "args": {
      "External id": 2479730,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297224.118, "dur": 0.357,
    "args": {
      "External id": 2479731,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297228.035, "dur": 0.495,
    "args": {
      "External id": 2479732,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297229.816, "dur": 0.597,
    "args": {
      "External id": 2479733,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297246.868, "dur": 24.632,
    "args": {
      "External id": 2479734,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297333.506, "dur": 382.473,
    "args": {
      "External id": 2479735,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297366.931, "dur": 343.872,
    "args": {
      "External id": 2479736,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7439, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297377.276, "dur": 327.082,
    "args": {
      "External id": 2479737,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297743.228, "dur": 2.446,
    "args": {
      "External id": 2479738,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7441, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297837.029, "dur": 17041.888,
    "args": {
      "External id": 2479739,"Record function id": 0, "Ev Idx": 7442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297941.976, "dur": 6.319,
    "args": {
      "External id": 2479740,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297951.896, "dur": 1.227,
    "args": {
      "External id": 2479741,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297954.744, "dur": 2.331,
    "args": {
      "External id": 2479742,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297958.869, "dur": 0.704,
    "args": {
      "External id": 2479743,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297961.022, "dur": 0.955,
    "args": {
      "External id": 2479744,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297963.277, "dur": 1.158,
    "args": {
      "External id": 2479745,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297968.095, "dur": 0.997,
    "args": {
      "External id": 2479746,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297970.548, "dur": 2.116,
    "args": {
      "External id": 2479747,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297974.574, "dur": 0.746,
    "args": {
      "External id": 2479748,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656297977.160, "dur": 0.828,
    "args": {
      "External id": 2479749,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298046.665, "dur": 16794.790,
    "args": {
      "External id": 2479750,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298063.937, "dur": 16771.275,
    "args": {
      "External id": 2479751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298091.401, "dur": 14.336,
    "args": {
      "External id": 2479752,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298108.747, "dur": 16693.189,
    "args": {
      "External id": 2479753,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298111.281, "dur": 16690.104,
    "args": {
      "External id": 2479754,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298117.191, "dur": 6.703,
    "args": {
      "External id": 2479755,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656298125.623, "dur": 16673.107,
    "args": {
      "External id": 2479756,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315051.129, "dur": 35.389,
    "args": {
      "External id": 2479757,"Sequence number": 24740457, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7460
    }
  },
  {
    "ph": "s", "id": 191, "pid": 1336760, "tid": 1336760, "ts": 1514656315051.129,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315073.492, "dur": 7.971,
    "args": {
      "External id": 2479758,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315076.722, "dur": 4.426,
    "args": {
      "External id": 2479759,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315153.343, "dur": 78.613,
    "args": {
      "External id": 2479760,"Record function id": 0, "Ev Idx": 7463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315233.603, "dur": 1120.624,
    "args": {
      "External id": 2479761,"Record function id": 0, "Ev Idx": 7464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315275.582, "dur": 1064.285,
    "args": {
      "External id": 2479762,"Sequence number": 24740458, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7465
    }
  },
  {
    "ph": "s", "id": 190, "pid": 1336760, "tid": 1336760, "ts": 1514656315275.582,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315348.479, "dur": 45.133,
    "args": {
      "External id": 2479763,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315408.288, "dur": 104.911,
    "args": {
      "External id": 2479764,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315524.805, "dur": 61.296,
    "args": {
      "External id": 2479765,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315596.062, "dur": 32.434,
    "args": {
      "External id": 2479766,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315657.746, "dur": 27.220,
    "args": {
      "External id": 2479767,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315701.743, "dur": 19.248,
    "args": {
      "External id": 2479768,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315737.270, "dur": 135.557,
    "args": {
      "External id": 2479769,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315789.107, "dur": 11.455,
    "args": {
      "External id": 2479770,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315794.070, "dur": 5.558,
    "args": {
      "External id": 2479771,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315803.771, "dur": 6.415,
    "args": {
      "External id": 2479772,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315811.450, "dur": 3.650,
    "args": {
      "External id": 2479773,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315817.411, "dur": 3.899,
    "args": {
      "External id": 2479774,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315882.952, "dur": 48.866,
    "args": {
      "External id": 2479775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656315963.542, "dur": 64.927,
    "args": {
      "External id": 2479776,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316041.964, "dur": 47.476,
    "args": {
      "External id": 2479777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316097.080, "dur": 36.321,
    "args": {
      "External id": 2479778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316159.948, "dur": 28.517,
    "args": {
      "External id": 2479779,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316194.481, "dur": 34.809,
    "args": {
      "External id": 2479780,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316247.895, "dur": 24.060,
    "args": {
      "External id": 2479781,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7484
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.18)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316417.771, "dur": 77.330,
    "args": {
      "External id": 2479782,"Record function id": 0, "Ev Idx": 7485
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316586.688, "dur": 48.099,
    "args": {
      "External id": 2479783,"Record function id": 0, "Ev Idx": 7486
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.19)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316644.346, "dur": 18334.745,
    "args": {
      "External id": 2479784,"Record function id": 0, "Ev Idx": 7487
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316652.775, "dur": 984.768,
    "args": {
      "External id": 2479785,"Record function id": 0, "Ev Idx": 7488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316737.307, "dur": 9.101,
    "args": {
      "External id": 2479786,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316760.674, "dur": 40.230,
    "args": {
      "External id": 2479787,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316766.379, "dur": 2.487,
    "args": {
      "External id": 2479788,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316773.492, "dur": 0.707,
    "args": {
      "External id": 2479789,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316775.599, "dur": 0.586,
    "args": {
      "External id": 2479790,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316777.765, "dur": 0.330,
    "args": {
      "External id": 2479791,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316781.915, "dur": 0.330,
    "args": {
      "External id": 2479792,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316784.215, "dur": 0.658,
    "args": {
      "External id": 2479793,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316786.985, "dur": 3.974,
    "args": {
      "External id": 2479794,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316792.304, "dur": 0.545,
    "args": {
      "External id": 2479795,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316794.301, "dur": 0.811,
    "args": {
      "External id": 2479796,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316813.765, "dur": 43.643,
    "args": {
      "External id": 2479797,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316889.860, "dur": 160.215,
    "args": {
      "External id": 2479798,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316900.811, "dur": 4.086,
    "args": {
      "External id": 2479799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316910.003, "dur": 10.825,
    "args": {
      "External id": 2479800,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316914.723, "dur": 5.681,
    "args": {
      "External id": 2479801,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316918.461, "dur": 0.724,
    "args": {
      "External id": 2479802,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316928.479, "dur": 36.317,
    "args": {
      "External id": 2479803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316930.976, "dur": 2.776,
    "args": {
      "External id": 2479804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316935.736, "dur": 0.653,
    "args": {
      "External id": 2479805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316938.039, "dur": 0.663,
    "args": {
      "External id": 2479806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316942.908, "dur": 1.887,
    "args": {
      "External id": 2479807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316946.929, "dur": 0.491,
    "args": {
      "External id": 2479808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316949.010, "dur": 0.495,
    "args": {
      "External id": 2479809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316952.973, "dur": 0.273,
    "args": {
      "External id": 2479810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316954.945, "dur": 0.450,
    "args": {
      "External id": 2479811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316956.850, "dur": 2.911,
    "args": {
      "External id": 2479812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656316978.469, "dur": 61.370,
    "args": {
      "External id": 2479813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317106.775, "dur": 408.905,
    "args": {
      "External id": 2479814,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317141.105, "dur": 368.921,
    "args": {
      "External id": 2479815,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7518, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317151.556, "dur": 349.941,
    "args": {
      "External id": 2479816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317560.819, "dur": 3.257,
    "args": {
      "External id": 2479817,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7520, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317659.716, "dur": 17118.265,
    "args": {
      "External id": 2479818,"Record function id": 0, "Ev Idx": 7521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317766.564, "dur": 7.119,
    "args": {
      "External id": 2479819,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317790.701, "dur": 1.509,
    "args": {
      "External id": 2479820,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317794.374, "dur": 2.938,
    "args": {
      "External id": 2479821,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317801.386, "dur": 1.014,
    "args": {
      "External id": 2479822,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317803.765, "dur": 0.630,
    "args": {
      "External id": 2479823,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317805.754, "dur": 1.061,
    "args": {
      "External id": 2479824,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317808.855, "dur": 0.979,
    "args": {
      "External id": 2479825,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317813.200, "dur": 2.052,
    "args": {
      "External id": 2479826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317816.808, "dur": 0.964,
    "args": {
      "External id": 2479827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317819.296, "dur": 1.029,
    "args": {
      "External id": 2479828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317840.326, "dur": 16898.197,
    "args": {
      "External id": 2479829,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317857.349, "dur": 16874.659,
    "args": {
      "External id": 2479830,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317877.605, "dur": 14.657,
    "args": {
      "External id": 2479831,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317897.160, "dur": 16799.627,
    "args": {
      "External id": 2479832,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317900.051, "dur": 16796.286,
    "args": {
      "External id": 2479833,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317906.435, "dur": 5.863,
    "args": {
      "External id": 2479834,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656317914.465, "dur": 16779.099,
    "args": {
      "External id": 2479835,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656334916.115, "dur": 39.222,
    "args": {
      "External id": 2479836,"Sequence number": 24740459, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7539
    }
  },
  {
    "ph": "s", "id": 189, "pid": 1336760, "tid": 1336760, "ts": 1514656334916.115,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656334942.205, "dur": 7.805,
    "args": {
      "External id": 2479837,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656334945.259, "dur": 4.562,
    "args": {
      "External id": 2479838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335059.015, "dur": 80.472,
    "args": {
      "External id": 2479839,"Record function id": 0, "Ev Idx": 7542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335140.974, "dur": 1118.329,
    "args": {
      "External id": 2479840,"Record function id": 0, "Ev Idx": 7543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335180.130, "dur": 1066.335,
    "args": {
      "External id": 2479841,"Sequence number": 24740460, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7544
    }
  },
  {
    "ph": "s", "id": 188, "pid": 1336760, "tid": 1336760, "ts": 1514656335180.130,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335249.573, "dur": 44.419,
    "args": {
      "External id": 2479842,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335307.316, "dur": 107.020,
    "args": {
      "External id": 2479843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335424.611, "dur": 39.233,
    "args": {
      "External id": 2479844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335472.796, "dur": 30.869,
    "args": {
      "External id": 2479845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335545.678, "dur": 30.954,
    "args": {
      "External id": 2479846,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335598.721, "dur": 16.705,
    "args": {
      "External id": 2479847,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335634.805, "dur": 133.590,
    "args": {
      "External id": 2479848,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335684.790, "dur": 12.947,
    "args": {
      "External id": 2479849,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335690.134, "dur": 6.858,
    "args": {
      "External id": 2479850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335700.785, "dur": 6.463,
    "args": {
      "External id": 2479851,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335708.588, "dur": 1.504,
    "args": {
      "External id": 2479852,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335712.470, "dur": 4.581,
    "args": {
      "External id": 2479853,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335779.111, "dur": 51.200,
    "args": {
      "External id": 2479854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335863.102, "dur": 29.131,
    "args": {
      "External id": 2479855,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335902.347, "dur": 41.823,
    "args": {
      "External id": 2479856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656335952.707, "dur": 72.408,
    "args": {
      "External id": 2479857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336053.393, "dur": 32.368,
    "args": {
      "External id": 2479858,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336092.581, "dur": 37.889,
    "args": {
      "External id": 2479859,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336151.804, "dur": 18.944,
    "args": {
      "External id": 2479860,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7563
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.19)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336323.352, "dur": 80.739,
    "args": {
      "External id": 2479861,"Record function id": 0, "Ev Idx": 7564
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336480.629, "dur": 62.405,
    "args": {
      "External id": 2479862,"Record function id": 0, "Ev Idx": 7565
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.20)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336555.502, "dur": 18227.539,
    "args": {
      "External id": 2479863,"Record function id": 0, "Ev Idx": 7566
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336566.319, "dur": 888.881,
    "args": {
      "External id": 2479864,"Record function id": 0, "Ev Idx": 7567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336653.390, "dur": 9.011,
    "args": {
      "External id": 2479865,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336676.299, "dur": 38.224,
    "args": {
      "External id": 2479866,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336681.784, "dur": 2.265,
    "args": {
      "External id": 2479867,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336689.077, "dur": 0.333,
    "args": {
      "External id": 2479868,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336691.460, "dur": 0.588,
    "args": {
      "External id": 2479869,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336693.339, "dur": 0.476,
    "args": {
      "External id": 2479870,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336696.860, "dur": 0.422,
    "args": {
      "External id": 2479871,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336698.533, "dur": 0.529,
    "args": {
      "External id": 2479872,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336700.432, "dur": 3.737,
    "args": {
      "External id": 2479873,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336705.746, "dur": 0.377,
    "args": {
      "External id": 2479874,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336707.610, "dur": 0.420,
    "args": {
      "External id": 2479875,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336726.892, "dur": 39.639,
    "args": {
      "External id": 2479876,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336799.586, "dur": 118.209,
    "args": {
      "External id": 2479877,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336810.278, "dur": 4.288,
    "args": {
      "External id": 2479878,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336819.767, "dur": 10.232,
    "args": {
      "External id": 2479879,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336824.309, "dur": 5.262,
    "args": {
      "External id": 2479880,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336827.800, "dur": 0.628,
    "args": {
      "External id": 2479881,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336836.939, "dur": 38.767,
    "args": {
      "External id": 2479882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336839.851, "dur": 2.782,
    "args": {
      "External id": 2479883,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336844.349, "dur": 0.613,
    "args": {
      "External id": 2479884,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336850.240, "dur": 0.630,
    "args": {
      "External id": 2479885,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336854.841, "dur": 2.047,
    "args": {
      "External id": 2479886,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336858.315, "dur": 0.328,
    "args": {
      "External id": 2479887,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336860.104, "dur": 2.906,
    "args": {
      "External id": 2479888,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336864.236, "dur": 0.598,
    "args": {
      "External id": 2479889,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336866.324, "dur": 0.520,
    "args": {
      "External id": 2479890,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336870.465, "dur": 0.342,
    "args": {
      "External id": 2479891,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336886.719, "dur": 23.532,
    "args": {
      "External id": 2479892,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656336970.861, "dur": 386.958,
    "args": {
      "External id": 2479893,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337041.223, "dur": 311.857,
    "args": {
      "External id": 2479894,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7597, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337053.192, "dur": 294.067,
    "args": {
      "External id": 2479895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337383.995, "dur": 2.086,
    "args": {
      "External id": 2479896,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7599, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337476.643, "dur": 17097.512,
    "args": {
      "External id": 2479897,"Record function id": 0, "Ev Idx": 7600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337596.240, "dur": 6.591,
    "args": {
      "External id": 2479898,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337606.672, "dur": 1.447,
    "args": {
      "External id": 2479899,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337609.815, "dur": 2.500,
    "args": {
      "External id": 2479900,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337614.399, "dur": 1.234,
    "args": {
      "External id": 2479901,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337617.026, "dur": 0.862,
    "args": {
      "External id": 2479902,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337619.226, "dur": 0.758,
    "args": {
      "External id": 2479903,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337623.977, "dur": 0.932,
    "args": {
      "External id": 2479904,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337626.646, "dur": 1.960,
    "args": {
      "External id": 2479905,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337630.195, "dur": 0.963,
    "args": {
      "External id": 2479906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337632.682, "dur": 0.547,
    "args": {
      "External id": 2479907,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337654.088, "dur": 16871.583,
    "args": {
      "External id": 2479908,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337670.322, "dur": 16848.975,
    "args": {
      "External id": 2479909,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337699.359, "dur": 13.621,
    "args": {
      "External id": 2479910,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337716.085, "dur": 16769.471,
    "args": {
      "External id": 2479911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337718.552, "dur": 16766.462,
    "args": {
      "External id": 2479912,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337725.125, "dur": 5.387,
    "args": {
      "External id": 2479913,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656337732.304, "dur": 16750.169,
    "args": {
      "External id": 2479914,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354719.337, "dur": 38.768,
    "args": {
      "External id": 2479915,"Sequence number": 24740461, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7618
    }
  },
  {
    "ph": "s", "id": 187, "pid": 1336760, "tid": 1336760, "ts": 1514656354719.337,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354743.734, "dur": 9.131,
    "args": {
      "External id": 2479916,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354748.112, "dur": 4.502,
    "args": {
      "External id": 2479917,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354823.568, "dur": 77.454,
    "args": {
      "External id": 2479918,"Record function id": 0, "Ev Idx": 7621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354902.686, "dur": 1144.495,
    "args": {
      "External id": 2479919,"Record function id": 0, "Ev Idx": 7622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656354944.467, "dur": 1087.581,
    "args": {
      "External id": 2479920,"Sequence number": 24740462, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7623
    }
  },
  {
    "ph": "s", "id": 186, "pid": 1336760, "tid": 1336760, "ts": 1514656354944.467,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355041.962, "dur": 46.199,
    "args": {
      "External id": 2479921,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355102.151, "dur": 106.456,
    "args": {
      "External id": 2479922,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355221.783, "dur": 42.408,
    "args": {
      "External id": 2479923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355270.837, "dur": 31.172,
    "args": {
      "External id": 2479924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355329.421, "dur": 28.038,
    "args": {
      "External id": 2479925,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355373.026, "dur": 17.349,
    "args": {
      "External id": 2479926,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355408.198, "dur": 153.169,
    "args": {
      "External id": 2479927,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355460.316, "dur": 12.391,
    "args": {
      "External id": 2479928,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355465.860, "dur": 6.115,
    "args": {
      "External id": 2479929,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355475.907, "dur": 5.543,
    "args": {
      "External id": 2479930,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355482.745, "dur": 1.564,
    "args": {
      "External id": 2479931,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355489.184, "dur": 4.796,
    "args": {
      "External id": 2479932,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355573.622, "dur": 52.471,
    "args": {
      "External id": 2479933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355660.407, "dur": 29.951,
    "args": {
      "External id": 2479934,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355699.934, "dur": 41.783,
    "args": {
      "External id": 2479935,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355748.907, "dur": 35.270,
    "args": {
      "External id": 2479936,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355808.257, "dur": 29.615,
    "args": {
      "External id": 2479937,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355843.045, "dur": 33.857,
    "args": {
      "External id": 2479938,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656355895.213, "dur": 19.797,
    "args": {
      "External id": 2479939,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.20)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356112.210, "dur": 79.380,
    "args": {
      "External id": 2479940,"Record function id": 0, "Ev Idx": 7643
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356266.678, "dur": 45.518,
    "args": {
      "External id": 2479941,"Record function id": 0, "Ev Idx": 7644
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.21)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356321.879, "dur": 18237.672,
    "args": {
      "External id": 2479942,"Record function id": 0, "Ev Idx": 7645
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356331.659, "dur": 952.339,
    "args": {
      "External id": 2479943,"Record function id": 0, "Ev Idx": 7646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356415.768, "dur": 8.886,
    "args": {
      "External id": 2479944,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356437.499, "dur": 40.579,
    "args": {
      "External id": 2479945,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356442.904, "dur": 2.449,
    "args": {
      "External id": 2479946,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356449.833, "dur": 0.666,
    "args": {
      "External id": 2479947,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356452.059, "dur": 0.454,
    "args": {
      "External id": 2479948,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356454.542, "dur": 0.682,
    "args": {
      "External id": 2479949,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356458.947, "dur": 0.547,
    "args": {
      "External id": 2479950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356461.050, "dur": 0.500,
    "args": {
      "External id": 2479951,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356463.102, "dur": 3.743,
    "args": {
      "External id": 2479952,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356468.480, "dur": 0.385,
    "args": {
      "External id": 2479953,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356470.779, "dur": 0.723,
    "args": {
      "External id": 2479954,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356490.911, "dur": 55.470,
    "args": {
      "External id": 2479955,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356582.359, "dur": 122.345,
    "args": {
      "External id": 2479956,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356594.070, "dur": 4.905,
    "args": {
      "External id": 2479957,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356604.611, "dur": 11.248,
    "args": {
      "External id": 2479958,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356609.210, "dur": 6.226,
    "args": {
      "External id": 2479959,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356613.278, "dur": 0.642,
    "args": {
      "External id": 2479960,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356623.298, "dur": 32.923,
    "args": {
      "External id": 2479961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356626.000, "dur": 2.700,
    "args": {
      "External id": 2479962,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356630.265, "dur": 0.412,
    "args": {
      "External id": 2479963,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356632.164, "dur": 0.419,
    "args": {
      "External id": 2479964,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356636.373, "dur": 2.276,
    "args": {
      "External id": 2479965,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356640.163, "dur": 0.185,
    "args": {
      "External id": 2479966,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356641.743, "dur": 0.380,
    "args": {
      "External id": 2479967,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356645.508, "dur": 0.289,
    "args": {
      "External id": 2479968,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356647.244, "dur": 0.632,
    "args": {
      "External id": 2479969,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356649.149, "dur": 2.630,
    "args": {
      "External id": 2479970,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356670.813, "dur": 26.587,
    "args": {
      "External id": 2479971,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356770.020, "dur": 410.514,
    "args": {
      "External id": 2479972,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356805.368, "dur": 369.944,
    "args": {
      "External id": 2479973,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7676, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656356815.869, "dur": 353.292,
    "args": {
      "External id": 2479974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357208.726, "dur": 2.360,
    "args": {
      "External id": 2479975,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7678, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357308.343, "dur": 17036.479,
    "args": {
      "External id": 2479976,"Record function id": 0, "Ev Idx": 7679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357416.885, "dur": 6.659,
    "args": {
      "External id": 2479977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357427.035, "dur": 1.411,
    "args": {
      "External id": 2479978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357430.246, "dur": 3.290,
    "args": {
      "External id": 2479979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357435.409, "dur": 1.186,
    "args": {
      "External id": 2479980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357438.011, "dur": 1.047,
    "args": {
      "External id": 2479981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357440.318, "dur": 1.021,
    "args": {
      "External id": 2479982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357445.102, "dur": 1.053,
    "args": {
      "External id": 2479983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357447.875, "dur": 1.975,
    "args": {
      "External id": 2479984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357451.315, "dur": 0.911,
    "args": {
      "External id": 2479985,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357453.921, "dur": 0.765,
    "args": {
      "External id": 2479986,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357474.921, "dur": 16831.767,
    "args": {
      "External id": 2479987,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357491.437, "dur": 16808.801,
    "args": {
      "External id": 2479988,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357514.346, "dur": 30.588,
    "args": {
      "External id": 2479989,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357548.685, "dur": 16717.832,
    "args": {
      "External id": 2479990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357551.225, "dur": 16714.796,
    "args": {
      "External id": 2479991,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357557.420, "dur": 6.601,
    "args": {
      "External id": 2479992,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656357565.911, "dur": 16697.401,
    "args": {
      "External id": 2479993,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374485.190, "dur": 34.597,
    "args": {
      "External id": 2479994,"Sequence number": 24740463, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7697
    }
  },
  {
    "ph": "s", "id": 185, "pid": 1336760, "tid": 1336760, "ts": 1514656374485.190,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374507.340, "dur": 6.989,
    "args": {
      "External id": 2479995,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374510.461, "dur": 3.718,
    "args": {
      "External id": 2479996,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374601.062, "dur": 78.323,
    "args": {
      "External id": 2479997,"Record function id": 0, "Ev Idx": 7700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374681.139, "dur": 1120.092,
    "args": {
      "External id": 2479998,"Record function id": 0, "Ev Idx": 7701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374723.473, "dur": 1064.438,
    "args": {
      "External id": 2479999,"Sequence number": 24740464, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7702
    }
  },
  {
    "ph": "s", "id": 184, "pid": 1336760, "tid": 1336760, "ts": 1514656374723.473,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374794.879, "dur": 42.758,
    "args": {
      "External id": 2480000,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374850.544, "dur": 107.236,
    "args": {
      "External id": 2480001,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656374967.860, "dur": 75.007,
    "args": {
      "External id": 2480002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375055.086, "dur": 34.753,
    "args": {
      "External id": 2480003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375118.428, "dur": 27.645,
    "args": {
      "External id": 2480004,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375165.272, "dur": 18.169,
    "args": {
      "External id": 2480005,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375202.705, "dur": 133.675,
    "args": {
      "External id": 2480006,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375253.621, "dur": 12.726,
    "args": {
      "External id": 2480007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375259.463, "dur": 6.118,
    "args": {
      "External id": 2480008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375269.285, "dur": 6.102,
    "args": {
      "External id": 2480009,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375276.734, "dur": 1.382,
    "args": {
      "External id": 2480010,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375280.682, "dur": 4.610,
    "args": {
      "External id": 2480011,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375347.380, "dur": 46.391,
    "args": {
      "External id": 2480012,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375425.193, "dur": 30.974,
    "args": {
      "External id": 2480013,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375466.044, "dur": 41.043,
    "args": {
      "External id": 2480014,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375516.363, "dur": 52.809,
    "args": {
      "External id": 2480015,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375597.855, "dur": 32.535,
    "args": {
      "External id": 2480016,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375637.201, "dur": 36.608,
    "args": {
      "External id": 2480017,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375694.526, "dur": 20.218,
    "args": {
      "External id": 2480018,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7721
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.21)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656375863.655, "dur": 75.828,
    "args": {
      "External id": 2480019,"Record function id": 0, "Ev Idx": 7722
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376052.673, "dur": 48.089,
    "args": {
      "External id": 2480020,"Record function id": 0, "Ev Idx": 7723
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.22)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376111.411, "dur": 18106.604,
    "args": {
      "External id": 2480021,"Record function id": 0, "Ev Idx": 7724
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376120.722, "dur": 911.608,
    "args": {
      "External id": 2480022,"Record function id": 0, "Ev Idx": 7725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376207.817, "dur": 9.101,
    "args": {
      "External id": 2480023,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376231.270, "dur": 41.350,
    "args": {
      "External id": 2480024,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376236.694, "dur": 2.567,
    "args": {
      "External id": 2480025,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376243.670, "dur": 0.436,
    "args": {
      "External id": 2480026,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376245.994, "dur": 0.589,
    "args": {
      "External id": 2480027,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376248.570, "dur": 0.387,
    "args": {
      "External id": 2480028,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376252.195, "dur": 0.596,
    "args": {
      "External id": 2480029,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376255.015, "dur": 0.424,
    "args": {
      "External id": 2480030,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376257.557, "dur": 3.795,
    "args": {
      "External id": 2480031,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376262.889, "dur": 0.603,
    "args": {
      "External id": 2480032,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376264.911, "dur": 0.639,
    "args": {
      "External id": 2480033,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376285.498, "dur": 42.832,
    "args": {
      "External id": 2480034,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376360.409, "dur": 121.022,
    "args": {
      "External id": 2480035,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376371.188, "dur": 4.074,
    "args": {
      "External id": 2480036,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376380.199, "dur": 10.312,
    "args": {
      "External id": 2480037,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376384.840, "dur": 5.235,
    "args": {
      "External id": 2480038,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376388.511, "dur": 0.440,
    "args": {
      "External id": 2480039,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376397.924, "dur": 37.476,
    "args": {
      "External id": 2480040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376400.621, "dur": 2.433,
    "args": {
      "External id": 2480041,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376404.741, "dur": 0.546,
    "args": {
      "External id": 2480042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376406.767, "dur": 0.440,
    "args": {
      "External id": 2480043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376411.274, "dur": 1.598,
    "args": {
      "External id": 2480044,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376414.700, "dur": 0.731,
    "args": {
      "External id": 2480045,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376416.660, "dur": 0.400,
    "args": {
      "External id": 2480046,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376420.603, "dur": 0.469,
    "args": {
      "External id": 2480047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376422.369, "dur": 0.160,
    "args": {
      "External id": 2480048,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376428.426, "dur": 2.399,
    "args": {
      "External id": 2480049,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376447.653, "dur": 25.748,
    "args": {
      "External id": 2480050,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376549.784, "dur": 352.366,
    "args": {
      "External id": 2480051,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376584.433, "dur": 312.509,
    "args": {
      "External id": 2480052,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7755, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376595.760, "dur": 292.538,
    "args": {
      "External id": 2480053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656376926.863, "dur": 2.252,
    "args": {
      "External id": 2480054,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7757, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377056.648, "dur": 16963.188,
    "args": {
      "External id": 2480055,"Record function id": 0, "Ev Idx": 7758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377158.108, "dur": 9.479,
    "args": {
      "External id": 2480056,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377170.868, "dur": 1.255,
    "args": {
      "External id": 2480057,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377173.809, "dur": 2.758,
    "args": {
      "External id": 2480058,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377178.358, "dur": 0.880,
    "args": {
      "External id": 2480059,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377180.429, "dur": 1.157,
    "args": {
      "External id": 2480060,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377182.657, "dur": 1.514,
    "args": {
      "External id": 2480061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377187.888, "dur": 1.232,
    "args": {
      "External id": 2480062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377190.585, "dur": 2.281,
    "args": {
      "External id": 2480063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377194.402, "dur": 0.978,
    "args": {
      "External id": 2480064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377196.748, "dur": 1.109,
    "args": {
      "External id": 2480065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377218.012, "dur": 16742.934,
    "args": {
      "External id": 2480066,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377243.053, "dur": 16711.261,
    "args": {
      "External id": 2480067,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377263.321, "dur": 14.447,
    "args": {
      "External id": 2480068,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377280.818, "dur": 16638.998,
    "args": {
      "External id": 2480069,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377283.508, "dur": 16635.664,
    "args": {
      "External id": 2480070,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377290.009, "dur": 6.216,
    "args": {
      "External id": 2480071,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656377297.960, "dur": 16618.530,
    "args": {
      "External id": 2480072,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394159.357, "dur": 34.841,
    "args": {
      "External id": 2480073,"Sequence number": 24740465, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7776
    }
  },
  {
    "ph": "s", "id": 183, "pid": 1336760, "tid": 1336760, "ts": 1514656394159.357,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394181.094, "dur": 8.152,
    "args": {
      "External id": 2480074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394184.426, "dur": 4.569,
    "args": {
      "External id": 2480075,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394257.855, "dur": 79.493,
    "args": {
      "External id": 2480076,"Record function id": 0, "Ev Idx": 7779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394338.873, "dur": 1114.638,
    "args": {
      "External id": 2480077,"Record function id": 0, "Ev Idx": 7780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394378.874, "dur": 1061.566,
    "args": {
      "External id": 2480078,"Sequence number": 24740466, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7781
    }
  },
  {
    "ph": "s", "id": 182, "pid": 1336760, "tid": 1336760, "ts": 1514656394378.874,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394444.949, "dur": 45.091,
    "args": {
      "External id": 2480079,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394502.792, "dur": 124.798,
    "args": {
      "External id": 2480080,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394640.875, "dur": 40.550,
    "args": {
      "External id": 2480081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394689.829, "dur": 31.656,
    "args": {
      "External id": 2480082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394747.364, "dur": 28.642,
    "args": {
      "External id": 2480083,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394793.672, "dur": 18.779,
    "args": {
      "External id": 2480084,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394830.237, "dur": 133.395,
    "args": {
      "External id": 2480085,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394882.615, "dur": 12.311,
    "args": {
      "External id": 2480086,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394888.320, "dur": 5.873,
    "args": {
      "External id": 2480087,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394898.258, "dur": 6.246,
    "args": {
      "External id": 2480088,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394906.080, "dur": 1.513,
    "args": {
      "External id": 2480089,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394910.501, "dur": 4.162,
    "args": {
      "External id": 2480090,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656394973.763, "dur": 90.155,
    "args": {
      "External id": 2480091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395101.376, "dur": 33.208,
    "args": {
      "External id": 2480092,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395144.735, "dur": 44.499,
    "args": {
      "External id": 2480093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395197.725, "dur": 34.830,
    "args": {
      "External id": 2480094,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395256.344, "dur": 30.893,
    "args": {
      "External id": 2480095,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395293.021, "dur": 34.725,
    "args": {
      "External id": 2480096,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395346.965, "dur": 19.985,
    "args": {
      "External id": 2480097,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7800
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.22)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395516.191, "dur": 96.133,
    "args": {
      "External id": 2480098,"Record function id": 0, "Ev Idx": 7801
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395688.032, "dur": 47.988,
    "args": {
      "External id": 2480099,"Record function id": 0, "Ev Idx": 7802
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.23)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395746.207, "dur": 18154.263,
    "args": {
      "External id": 2480100,"Record function id": 0, "Ev Idx": 7803
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395754.332, "dur": 977.016,
    "args": {
      "External id": 2480101,"Record function id": 0, "Ev Idx": 7804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395842.094, "dur": 8.596,
    "args": {
      "External id": 2480102,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395864.638, "dur": 40.519,
    "args": {
      "External id": 2480103,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395870.180, "dur": 2.486,
    "args": {
      "External id": 2480104,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395877.555, "dur": 0.418,
    "args": {
      "External id": 2480105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395879.310, "dur": 0.772,
    "args": {
      "External id": 2480106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395882.238, "dur": 0.244,
    "args": {
      "External id": 2480107,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395886.273, "dur": 0.444,
    "args": {
      "External id": 2480108,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395888.092, "dur": 0.630,
    "args": {
      "External id": 2480109,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395890.271, "dur": 3.560,
    "args": {
      "External id": 2480110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395895.759, "dur": 0.199,
    "args": {
      "External id": 2480111,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395897.487, "dur": 0.479,
    "args": {
      "External id": 2480112,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656395917.570, "dur": 43.807,
    "args": {
      "External id": 2480113,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396031.812, "dur": 153.748,
    "args": {
      "External id": 2480114,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396043.853, "dur": 5.704,
    "args": {
      "External id": 2480115,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396055.198, "dur": 11.159,
    "args": {
      "External id": 2480116,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396060.041, "dur": 5.894,
    "args": {
      "External id": 2480117,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396063.948, "dur": 0.568,
    "args": {
      "External id": 2480118,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396074.396, "dur": 30.830,
    "args": {
      "External id": 2480119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396076.932, "dur": 2.673,
    "args": {
      "External id": 2480120,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396081.481, "dur": 0.333,
    "args": {
      "External id": 2480121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396083.337, "dur": 0.414,
    "args": {
      "External id": 2480122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396087.294, "dur": 2.378,
    "args": {
      "External id": 2480123,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396091.097, "dur": 0.619,
    "args": {
      "External id": 2480124,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396092.621, "dur": 0.560,
    "args": {
      "External id": 2480125,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396094.671, "dur": 0.456,
    "args": {
      "External id": 2480126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396096.628, "dur": 0.153,
    "args": {
      "External id": 2480127,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396098.115, "dur": 2.241,
    "args": {
      "External id": 2480128,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396146.715, "dur": 30.183,
    "args": {
      "External id": 2480129,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396241.800, "dur": 388.348,
    "args": {
      "External id": 2480130,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396274.222, "dur": 350.545,
    "args": {
      "External id": 2480131,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7834, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396286.857, "dur": 328.533,
    "args": {
      "External id": 2480132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396655.499, "dur": 2.219,
    "args": {
      "External id": 2480133,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7836, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396756.194, "dur": 16947.510,
    "args": {
      "External id": 2480134,"Record function id": 0, "Ev Idx": 7837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396861.028, "dur": 6.312,
    "args": {
      "External id": 2480135,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396870.950, "dur": 1.351,
    "args": {
      "External id": 2480136,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396873.872, "dur": 2.833,
    "args": {
      "External id": 2480137,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396878.316, "dur": 0.891,
    "args": {
      "External id": 2480138,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396880.929, "dur": 1.339,
    "args": {
      "External id": 2480139,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396883.744, "dur": 1.100,
    "args": {
      "External id": 2480140,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396888.365, "dur": 0.833,
    "args": {
      "External id": 2480141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396890.767, "dur": 1.617,
    "args": {
      "External id": 2480142,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396893.665, "dur": 0.924,
    "args": {
      "External id": 2480143,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396896.085, "dur": 0.870,
    "args": {
      "External id": 2480144,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396917.926, "dur": 16749.240,
    "args": {
      "External id": 2480145,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396934.314, "dur": 16726.288,
    "args": {
      "External id": 2480146,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396957.600, "dur": 14.447,
    "args": {
      "External id": 2480147,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396975.044, "dur": 16652.697,
    "args": {
      "External id": 2480148,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656396977.526, "dur": 16649.755,
    "args": {
      "External id": 2480149,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656397018.177, "dur": 6.635,
    "args": {
      "External id": 2480150,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656397027.122, "dur": 16597.371,
    "args": {
      "External id": 2480151,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656413841.151, "dur": 35.674,
    "args": {
      "External id": 2480152,"Sequence number": 24740467, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7855
    }
  },
  {
    "ph": "s", "id": 181, "pid": 1336760, "tid": 1336760, "ts": 1514656413841.151,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656413864.036, "dur": 7.750,
    "args": {
      "External id": 2480153,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656413867.205, "dur": 4.400,
    "args": {
      "External id": 2480154,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656413939.773, "dur": 107.807,
    "args": {
      "External id": 2480155,"Record function id": 0, "Ev Idx": 7858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414050.642, "dur": 1101.960,
    "args": {
      "External id": 2480156,"Record function id": 0, "Ev Idx": 7859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414096.879, "dur": 1043.151,
    "args": {
      "External id": 2480157,"Sequence number": 24740468, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7860
    }
  },
  {
    "ph": "s", "id": 180, "pid": 1336760, "tid": 1336760, "ts": 1514656414096.879,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414163.020, "dur": 44.685,
    "args": {
      "External id": 2480158,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414220.660, "dur": 103.575,
    "args": {
      "External id": 2480159,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414334.464, "dur": 39.754,
    "args": {
      "External id": 2480160,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414383.070, "dur": 32.419,
    "args": {
      "External id": 2480161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414439.941, "dur": 28.002,
    "args": {
      "External id": 2480162,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414485.979, "dur": 18.453,
    "args": {
      "External id": 2480163,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414521.979, "dur": 149.560,
    "args": {
      "External id": 2480164,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414587.240, "dur": 12.624,
    "args": {
      "External id": 2480165,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414592.437, "dur": 6.521,
    "args": {
      "External id": 2480166,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414602.749, "dur": 6.402,
    "args": {
      "External id": 2480167,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414610.318, "dur": 1.336,
    "args": {
      "External id": 2480168,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414614.497, "dur": 4.640,
    "args": {
      "External id": 2480169,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414681.973, "dur": 51.417,
    "args": {
      "External id": 2480170,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414765.792, "dur": 30.053,
    "args": {
      "External id": 2480171,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414804.971, "dur": 43.333,
    "args": {
      "External id": 2480172,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414857.021, "dur": 35.206,
    "args": {
      "External id": 2480173,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414915.670, "dur": 28.277,
    "args": {
      "External id": 2480174,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656414949.349, "dur": 67.961,
    "args": {
      "External id": 2480175,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415041.909, "dur": 21.135,
    "args": {
      "External id": 2480176,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7879
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.23)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415214.237, "dur": 77.933,
    "args": {
      "External id": 2480177,"Record function id": 0, "Ev Idx": 7880
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415365.756, "dur": 46.827,
    "args": {
      "External id": 2480178,"Record function id": 0, "Ev Idx": 7881
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.24)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415422.065, "dur": 18161.723,
    "args": {
      "External id": 2480179,"Record function id": 0, "Ev Idx": 7882
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415431.526, "dur": 925.058,
    "args": {
      "External id": 2480180,"Record function id": 0, "Ev Idx": 7883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415518.739, "dur": 23.672,
    "args": {
      "External id": 2480181,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415558.026, "dur": 34.732,
    "args": {
      "External id": 2480182,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415562.926, "dur": 2.680,
    "args": {
      "External id": 2480183,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415570.126, "dur": 0.433,
    "args": {
      "External id": 2480184,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415571.739, "dur": 0.299,
    "args": {
      "External id": 2480185,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415572.905, "dur": 0.575,
    "args": {
      "External id": 2480186,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415576.379, "dur": 0.549,
    "args": {
      "External id": 2480187,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415577.938, "dur": 0.466,
    "args": {
      "External id": 2480188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415579.557, "dur": 3.670,
    "args": {
      "External id": 2480189,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415584.325, "dur": 0.385,
    "args": {
      "External id": 2480190,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415585.724, "dur": 0.256,
    "args": {
      "External id": 2480191,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415606.249, "dur": 41.469,
    "args": {
      "External id": 2480192,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415680.522, "dur": 114.093,
    "args": {
      "External id": 2480193,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415691.700, "dur": 5.129,
    "args": {
      "External id": 2480194,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415702.193, "dur": 11.093,
    "args": {
      "External id": 2480195,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415707.229, "dur": 5.657,
    "args": {
      "External id": 2480196,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415711.166, "dur": 0.396,
    "args": {
      "External id": 2480197,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415720.090, "dur": 31.720,
    "args": {
      "External id": 2480198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415722.125, "dur": 2.363,
    "args": {
      "External id": 2480199,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415726.498, "dur": 0.503,
    "args": {
      "External id": 2480200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415727.977, "dur": 0.518,
    "args": {
      "External id": 2480201,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415731.865, "dur": 1.668,
    "args": {
      "External id": 2480202,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415734.785, "dur": 0.423,
    "args": {
      "External id": 2480203,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415736.618, "dur": 0.428,
    "args": {
      "External id": 2480204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415740.213, "dur": 0.259,
    "args": {
      "External id": 2480205,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415741.722, "dur": 0.320,
    "args": {
      "External id": 2480206,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415743.413, "dur": 2.762,
    "args": {
      "External id": 2480207,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415762.502, "dur": 23.891,
    "args": {
      "External id": 2480208,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415850.029, "dur": 405.923,
    "args": {
      "External id": 2480209,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415890.264, "dur": 360.046,
    "args": {
      "External id": 2480210,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7913, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656415901.045, "dur": 342.722,
    "args": {
      "External id": 2480211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416280.343, "dur": 2.228,
    "args": {
      "External id": 2480212,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7915, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416381.809, "dur": 16982.295,
    "args": {
      "External id": 2480213,"Record function id": 0, "Ev Idx": 7916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416494.788, "dur": 6.241,
    "args": {
      "External id": 2480214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416510.346, "dur": 1.387,
    "args": {
      "External id": 2480215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416513.278, "dur": 2.362,
    "args": {
      "External id": 2480216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416518.806, "dur": 0.948,
    "args": {
      "External id": 2480217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416521.045, "dur": 0.921,
    "args": {
      "External id": 2480218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416523.061, "dur": 1.098,
    "args": {
      "External id": 2480219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416542.005, "dur": 1.827,
    "args": {
      "External id": 2480220,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416549.601, "dur": 1.925,
    "args": {
      "External id": 2480221,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416552.815, "dur": 0.975,
    "args": {
      "External id": 2480222,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416555.301, "dur": 0.557,
    "args": {
      "External id": 2480223,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416577.008, "dur": 16747.374,
    "args": {
      "External id": 2480224,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416593.347, "dur": 16724.478,
    "args": {
      "External id": 2480225,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416614.242, "dur": 13.692,
    "args": {
      "External id": 2480226,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416632.792, "dur": 16650.499,
    "args": {
      "External id": 2480227,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416635.236, "dur": 16647.530,
    "args": {
      "External id": 2480228,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416640.974, "dur": 5.933,
    "args": {
      "External id": 2480229,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656416648.642, "dur": 16631.230,
    "args": {
      "External id": 2480230,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433507.761, "dur": 49.732,
    "args": {
      "External id": 2480231,"Sequence number": 24740469, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7934
    }
  },
  {
    "ph": "s", "id": 179, "pid": 1336760, "tid": 1336760, "ts": 1514656433507.761,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433543.498, "dur": 8.350,
    "args": {
      "External id": 2480232,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433546.927, "dur": 4.577,
    "args": {
      "External id": 2480233,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433625.236, "dur": 79.101,
    "args": {
      "External id": 2480234,"Record function id": 0, "Ev Idx": 7937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433705.818, "dur": 1107.488,
    "args": {
      "External id": 2480235,"Record function id": 0, "Ev Idx": 7938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433745.427, "dur": 1054.240,
    "args": {
      "External id": 2480236,"Sequence number": 24740470, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7939
    }
  },
  {
    "ph": "s", "id": 178, "pid": 1336760, "tid": 1336760, "ts": 1514656433745.427,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433814.304, "dur": 44.098,
    "args": {
      "External id": 2480237,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656433870.517, "dur": 106.664,
    "args": {
      "External id": 2480238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434026.109, "dur": 45.603,
    "args": {
      "External id": 2480239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434081.776, "dur": 32.158,
    "args": {
      "External id": 2480240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434141.085, "dur": 30.682,
    "args": {
      "External id": 2480241,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434189.906, "dur": 15.489,
    "args": {
      "External id": 2480242,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434223.731, "dur": 129.839,
    "args": {
      "External id": 2480243,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434274.514, "dur": 11.287,
    "args": {
      "External id": 2480244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434279.482, "dur": 5.541,
    "args": {
      "External id": 2480245,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434288.817, "dur": 5.197,
    "args": {
      "External id": 2480246,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434295.092, "dur": 1.257,
    "args": {
      "External id": 2480247,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434298.671, "dur": 4.023,
    "args": {
      "External id": 2480248,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434364.164, "dur": 46.779,
    "args": {
      "External id": 2480249,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434442.223, "dur": 28.894,
    "args": {
      "External id": 2480250,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434480.193, "dur": 41.702,
    "args": {
      "External id": 2480251,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434545.766, "dur": 40.378,
    "args": {
      "External id": 2480252,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434624.160, "dur": 28.882,
    "args": {
      "External id": 2480253,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434661.116, "dur": 35.236,
    "args": {
      "External id": 2480254,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434714.086, "dur": 20.095,
    "args": {
      "External id": 2480255,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7958
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.24)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656434877.439, "dur": 80.228,
    "args": {
      "External id": 2480256,"Record function id": 0, "Ev Idx": 7959
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435069.816, "dur": 47.836,
    "args": {
      "External id": 2480257,"Record function id": 0, "Ev Idx": 7960
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.25)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435127.694, "dur": 18261.853,
    "args": {
      "External id": 2480258,"Record function id": 0, "Ev Idx": 7961
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435137.966, "dur": 961.237,
    "args": {
      "External id": 2480259,"Record function id": 0, "Ev Idx": 7962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435224.848, "dur": 9.439,
    "args": {
      "External id": 2480260,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435248.081, "dur": 35.623,
    "args": {
      "External id": 2480261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435253.028, "dur": 2.367,
    "args": {
      "External id": 2480262,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435259.953, "dur": 0.428,
    "args": {
      "External id": 2480263,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435261.657, "dur": 0.318,
    "args": {
      "External id": 2480264,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435263.261, "dur": 0.632,
    "args": {
      "External id": 2480265,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435267.159, "dur": 0.593,
    "args": {
      "External id": 2480266,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435268.610, "dur": 0.558,
    "args": {
      "External id": 2480267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435269.934, "dur": 3.696,
    "args": {
      "External id": 2480268,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435274.731, "dur": 0.272,
    "args": {
      "External id": 2480269,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435276.044, "dur": 0.393,
    "args": {
      "External id": 2480270,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435296.774, "dur": 45.465,
    "args": {
      "External id": 2480271,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435374.074, "dur": 113.708,
    "args": {
      "External id": 2480272,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435384.920, "dur": 4.419,
    "args": {
      "External id": 2480273,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435394.437, "dur": 10.695,
    "args": {
      "External id": 2480274,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435399.209, "dur": 5.484,
    "args": {
      "External id": 2480275,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435402.761, "dur": 0.573,
    "args": {
      "External id": 2480276,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435412.178, "dur": 31.092,
    "args": {
      "External id": 2480277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435413.812, "dur": 2.634,
    "args": {
      "External id": 2480278,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435418.045, "dur": 0.399,
    "args": {
      "External id": 2480279,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435419.341, "dur": 0.221,
    "args": {
      "External id": 2480280,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435422.881, "dur": 2.291,
    "args": {
      "External id": 2480281,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435425.871, "dur": 0.542,
    "args": {
      "External id": 2480282,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435427.613, "dur": 0.402,
    "args": {
      "External id": 2480283,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435430.845, "dur": 0.565,
    "args": {
      "External id": 2480284,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435432.582, "dur": 0.367,
    "args": {
      "External id": 2480285,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435433.822, "dur": 2.776,
    "args": {
      "External id": 2480286,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435456.399, "dur": 23.668,
    "args": {
      "External id": 2480287,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435557.136, "dur": 388.644,
    "args": {
      "External id": 2480288,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435591.208, "dur": 349.925,
    "args": {
      "External id": 2480289,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7992, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435602.181, "dur": 333.393,
    "args": {
      "External id": 2480290,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656435968.509, "dur": 2.107,
    "args": {
      "External id": 2480291,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7994, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436121.138, "dur": 17077.763,
    "args": {
      "External id": 2480292,"Record function id": 0, "Ev Idx": 7995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436227.753, "dur": 6.390,
    "args": {
      "External id": 2480293,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436237.551, "dur": 1.324,
    "args": {
      "External id": 2480294,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436240.417, "dur": 2.679,
    "args": {
      "External id": 2480295,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436244.683, "dur": 1.153,
    "args": {
      "External id": 2480296,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436246.964, "dur": 1.034,
    "args": {
      "External id": 2480297,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436249.205, "dur": 1.049,
    "args": {
      "External id": 2480298,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436257.685, "dur": 0.855,
    "args": {
      "External id": 2480299,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436259.931, "dur": 2.344,
    "args": {
      "External id": 2480300,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436263.711, "dur": 0.768,
    "args": {
      "External id": 2480301,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436265.611, "dur": 0.726,
    "args": {
      "External id": 2480302,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436286.165, "dur": 16876.297,
    "args": {
      "External id": 2480303,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436301.984, "dur": 16854.007,
    "args": {
      "External id": 2480304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436329.460, "dur": 15.292,
    "args": {
      "External id": 2480305,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436347.580, "dur": 16775.232,
    "args": {
      "External id": 2480306,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436350.073, "dur": 16772.291,
    "args": {
      "External id": 2480307,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436356.937, "dur": 6.185,
    "args": {
      "External id": 2480308,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656436364.697, "dur": 16754.694,
    "args": {
      "External id": 2480309,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453332.390, "dur": 34.214,
    "args": {
      "External id": 2480310,"Sequence number": 24740471, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8013
    }
  },
  {
    "ph": "s", "id": 177, "pid": 1336760, "tid": 1336760, "ts": 1514656453332.390,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453354.013, "dur": 7.441,
    "args": {
      "External id": 2480311,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453357.154, "dur": 4.088,
    "args": {
      "External id": 2480312,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453429.233, "dur": 82.377,
    "args": {
      "External id": 2480313,"Record function id": 0, "Ev Idx": 8016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453513.132, "dur": 1133.400,
    "args": {
      "External id": 2480314,"Record function id": 0, "Ev Idx": 8017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453568.013, "dur": 1064.782,
    "args": {
      "External id": 2480315,"Sequence number": 24740472, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8018
    }
  },
  {
    "ph": "s", "id": 176, "pid": 1336760, "tid": 1336760, "ts": 1514656453568.013,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453639.334, "dur": 45.211,
    "args": {
      "External id": 2480316,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453697.716, "dur": 108.804,
    "args": {
      "External id": 2480317,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453816.283, "dur": 39.478,
    "args": {
      "External id": 2480318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453865.479, "dur": 32.704,
    "args": {
      "External id": 2480319,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453924.082, "dur": 29.217,
    "args": {
      "External id": 2480320,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656453972.249, "dur": 52.720,
    "args": {
      "External id": 2480321,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454045.408, "dur": 132.372,
    "args": {
      "External id": 2480322,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454097.085, "dur": 12.412,
    "args": {
      "External id": 2480323,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454102.316, "dur": 6.400,
    "args": {
      "External id": 2480324,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454112.456, "dur": 4.684,
    "args": {
      "External id": 2480325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454118.429, "dur": 1.368,
    "args": {
      "External id": 2480326,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454122.281, "dur": 4.300,
    "args": {
      "External id": 2480327,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454188.267, "dur": 53.260,
    "args": {
      "External id": 2480328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454273.095, "dur": 29.478,
    "args": {
      "External id": 2480329,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454311.579, "dur": 42.677,
    "args": {
      "External id": 2480330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454363.152, "dur": 35.841,
    "args": {
      "External id": 2480331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454422.068, "dur": 43.146,
    "args": {
      "External id": 2480332,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454470.630, "dur": 34.291,
    "args": {
      "External id": 2480333,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454524.195, "dur": 35.380,
    "args": {
      "External id": 2480334,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8037
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.25)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454710.916, "dur": 73.746,
    "args": {
      "External id": 2480335,"Record function id": 0, "Ev Idx": 8038
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454859.646, "dur": 48.325,
    "args": {
      "External id": 2480336,"Record function id": 0, "Ev Idx": 8039
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.26)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454917.561, "dur": 18272.505,
    "args": {
      "External id": 2480337,"Record function id": 0, "Ev Idx": 8040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656454927.323, "dur": 911.878,
    "args": {
      "External id": 2480338,"Record function id": 0, "Ev Idx": 8041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455047.573, "dur": 10.563,
    "args": {
      "External id": 2480339,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455072.059, "dur": 36.148,
    "args": {
      "External id": 2480340,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455077.839, "dur": 2.156,
    "args": {
      "External id": 2480341,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455084.766, "dur": 0.210,
    "args": {
      "External id": 2480342,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455086.738, "dur": 0.356,
    "args": {
      "External id": 2480343,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455088.316, "dur": 0.210,
    "args": {
      "External id": 2480344,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455091.519, "dur": 0.317,
    "args": {
      "External id": 2480345,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455093.054, "dur": 0.335,
    "args": {
      "External id": 2480346,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455094.614, "dur": 2.738,
    "args": {
      "External id": 2480347,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455098.381, "dur": 1.441,
    "args": {
      "External id": 2480348,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455101.201, "dur": 0.201,
    "args": {
      "External id": 2480349,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455120.830, "dur": 40.751,
    "args": {
      "External id": 2480350,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455193.418, "dur": 110.632,
    "args": {
      "External id": 2480351,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455204.613, "dur": 3.966,
    "args": {
      "External id": 2480352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455213.567, "dur": 9.795,
    "args": {
      "External id": 2480353,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455217.742, "dur": 5.187,
    "args": {
      "External id": 2480354,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455221.089, "dur": 0.543,
    "args": {
      "External id": 2480355,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455230.195, "dur": 32.018,
    "args": {
      "External id": 2480356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455232.305, "dur": 2.792,
    "args": {
      "External id": 2480357,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455236.527, "dur": 0.520,
    "args": {
      "External id": 2480358,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455241.808, "dur": 0.294,
    "args": {
      "External id": 2480359,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455246.387, "dur": 0.278,
    "args": {
      "External id": 2480360,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455247.877, "dur": 0.953,
    "args": {
      "External id": 2480361,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455249.621, "dur": 2.141,
    "args": {
      "External id": 2480362,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455252.727, "dur": 0.445,
    "args": {
      "External id": 2480363,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455254.179, "dur": 0.241,
    "args": {
      "External id": 2480364,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455257.261, "dur": 0.178,
    "args": {
      "External id": 2480365,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455274.376, "dur": 21.315,
    "args": {
      "External id": 2480366,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455357.573, "dur": 387.502,
    "args": {
      "External id": 2480367,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455389.894, "dur": 349.664,
    "args": {
      "External id": 2480368,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8071, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455400.579, "dur": 332.664,
    "args": {
      "External id": 2480369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455768.726, "dur": 2.183,
    "args": {
      "External id": 2480370,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8073, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455859.608, "dur": 17092.343,
    "args": {
      "External id": 2480371,"Record function id": 0, "Ev Idx": 8074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455966.118, "dur": 6.542,
    "args": {
      "External id": 2480372,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656455975.625, "dur": 0.926,
    "args": {
      "External id": 2480373,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456018.586, "dur": 1.871,
    "args": {
      "External id": 2480374,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456026.107, "dur": 1.441,
    "args": {
      "External id": 2480375,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456029.030, "dur": 1.476,
    "args": {
      "External id": 2480376,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456031.797, "dur": 1.296,
    "args": {
      "External id": 2480377,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456034.766, "dur": 1.162,
    "args": {
      "External id": 2480378,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456039.373, "dur": 2.385,
    "args": {
      "External id": 2480379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456043.203, "dur": 1.031,
    "args": {
      "External id": 2480380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456045.691, "dur": 1.003,
    "args": {
      "External id": 2480381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456067.696, "dur": 16845.409,
    "args": {
      "External id": 2480382,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456082.995, "dur": 16823.472,
    "args": {
      "External id": 2480383,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456102.160, "dur": 13.282,
    "args": {
      "External id": 2480384,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456120.766, "dur": 16751.990,
    "args": {
      "External id": 2480385,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456123.339, "dur": 16748.879,
    "args": {
      "External id": 2480386,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456129.206, "dur": 6.820,
    "args": {
      "External id": 2480387,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656456137.974, "dur": 16731.334,
    "args": {
      "External id": 2480388,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473129.586, "dur": 35.934,
    "args": {
      "External id": 2480389,"Sequence number": 24740473, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8092
    }
  },
  {
    "ph": "s", "id": 175, "pid": 1336760, "tid": 1336760, "ts": 1514656473129.586,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473152.809, "dur": 7.737,
    "args": {
      "External id": 2480390,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473156.051, "dur": 4.152,
    "args": {
      "External id": 2480391,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473230.311, "dur": 78.010,
    "args": {
      "External id": 2480392,"Record function id": 0, "Ev Idx": 8095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473310.145, "dur": 1099.137,
    "args": {
      "External id": 2480393,"Record function id": 0, "Ev Idx": 8096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473348.767, "dur": 1046.928,
    "args": {
      "External id": 2480394,"Sequence number": 24740474, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8097
    }
  },
  {
    "ph": "s", "id": 174, "pid": 1336760, "tid": 1336760, "ts": 1514656473348.767,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473417.329, "dur": 44.695,
    "args": {
      "External id": 2480395,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473475.477, "dur": 124.322,
    "args": {
      "External id": 2480396,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473612.596, "dur": 41.897,
    "args": {
      "External id": 2480397,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473663.970, "dur": 31.614,
    "args": {
      "External id": 2480398,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473720.566, "dur": 28.725,
    "args": {
      "External id": 2480399,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473769.419, "dur": 15.413,
    "args": {
      "External id": 2480400,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473802.130, "dur": 125.805,
    "args": {
      "External id": 2480401,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473851.175, "dur": 11.942,
    "args": {
      "External id": 2480402,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473856.431, "dur": 5.976,
    "args": {
      "External id": 2480403,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473865.738, "dur": 5.035,
    "args": {
      "External id": 2480404,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473871.935, "dur": 1.440,
    "args": {
      "External id": 2480405,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473875.871, "dur": 3.476,
    "args": {
      "External id": 2480406,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656473938.415, "dur": 85.877,
    "args": {
      "External id": 2480407,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474061.261, "dur": 29.525,
    "args": {
      "External id": 2480408,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474099.569, "dur": 46.340,
    "args": {
      "External id": 2480409,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474155.272, "dur": 36.767,
    "args": {
      "External id": 2480410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474214.456, "dur": 27.876,
    "args": {
      "External id": 2480411,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474248.243, "dur": 35.680,
    "args": {
      "External id": 2480412,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474305.535, "dur": 18.416,
    "args": {
      "External id": 2480413,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8116
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.26)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474474.341, "dur": 96.123,
    "args": {
      "External id": 2480414,"Record function id": 0, "Ev Idx": 8117
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474648.881, "dur": 49.419,
    "args": {
      "External id": 2480415,"Record function id": 0, "Ev Idx": 8118
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.27)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474707.302, "dur": 18165.303,
    "args": {
      "External id": 2480416,"Record function id": 0, "Ev Idx": 8119
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.27)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474715.865, "dur": 903.927,
    "args": {
      "External id": 2480417,"Record function id": 0, "Ev Idx": 8120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474801.161, "dur": 9.005,
    "args": {
      "External id": 2480418,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474823.197, "dur": 34.325,
    "args": {
      "External id": 2480419,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474828.110, "dur": 2.621,
    "args": {
      "External id": 2480420,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474834.872, "dur": 0.636,
    "args": {
      "External id": 2480421,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474837.041, "dur": 0.468,
    "args": {
      "External id": 2480422,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474838.709, "dur": 0.881,
    "args": {
      "External id": 2480423,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474842.482, "dur": 1.209,
    "args": {
      "External id": 2480424,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474844.835, "dur": 0.521,
    "args": {
      "External id": 2480425,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474846.405, "dur": 2.170,
    "args": {
      "External id": 2480426,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474849.598, "dur": 0.661,
    "args": {
      "External id": 2480427,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474851.087, "dur": 0.586,
    "args": {
      "External id": 2480428,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474870.339, "dur": 40.115,
    "args": {
      "External id": 2480429,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474940.774, "dur": 158.617,
    "args": {
      "External id": 2480430,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "7", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474951.334, "dur": 4.193,
    "args": {
      "External id": 2480431,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474960.606, "dur": 9.950,
    "args": {
      "External id": 2480432,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474965.024, "dur": 5.086,
    "args": {
      "External id": 2480433,"Record function id": 0, "Concrete Inputs": ["", "0", "44961280", "51384320", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474968.194, "dur": 0.850,
    "args": {
      "External id": 2480434,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474977.240, "dur": 69.058,
    "args": {
      "External id": 2480435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656474979.503, "dur": 37.974,
    "args": {
      "External id": 2480436,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "44961280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475020.733, "dur": 0.657,
    "args": {
      "External id": 2480437,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "44961536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475023.152, "dur": 0.564,
    "args": {
      "External id": 2480438,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "45485824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475027.101, "dur": 0.554,
    "args": {
      "External id": 2480439,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46010112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475029.096, "dur": 0.629,
    "args": {
      "External id": 2480440,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "46534400"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475031.089, "dur": 0.618,
    "args": {
      "External id": 2480441,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "47058688"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475035.160, "dur": 0.684,
    "args": {
      "External id": 2480442,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "47058944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475036.909, "dur": 0.562,
    "args": {
      "External id": 2480443,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "48500736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475038.968, "dur": 2.634,
    "args": {
      "External id": 2480444,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "49942528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475061.348, "dur": 29.376,
    "args": {
      "External id": 2480445,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475157.071, "dur": 353.813,
    "args": {
      "External id": 2480446,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475190.062, "dur": 316.017,
    "args": {
      "External id": 2480447,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 7, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8150, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475200.540, "dur": 297.007,
    "args": {
      "External id": 2480448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475548.949, "dur": 3.430,
    "args": {
      "External id": 2480449,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8152, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.27)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475640.303, "dur": 17035.688,
    "args": {
      "External id": 2480450,"Record function id": 0, "Ev Idx": 8153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475763.528, "dur": 6.618,
    "args": {
      "External id": 2480451,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475773.673, "dur": 0.881,
    "args": {
      "External id": 2480452,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475776.052, "dur": 1.447,
    "args": {
      "External id": 2480453,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475781.186, "dur": 0.873,
    "args": {
      "External id": 2480454,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475783.464, "dur": 1.316,
    "args": {
      "External id": 2480455,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475786.068, "dur": 1.135,
    "args": {
      "External id": 2480456,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475788.559, "dur": 1.010,
    "args": {
      "External id": 2480457,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475792.993, "dur": 1.990,
    "args": {
      "External id": 2480458,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475796.204, "dur": 2.100,
    "args": {
      "External id": 2480459,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475799.651, "dur": 1.019,
    "args": {
      "External id": 2480460,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475819.234, "dur": 16819.867,
    "args": {
      "External id": 2480461,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475835.707, "dur": 16796.533,
    "args": {
      "External id": 2480462,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475853.523, "dur": 13.828,
    "args": {
      "External id": 2480463,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475872.366, "dur": 16726.275,
    "args": {
      "External id": 2480464,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475874.903, "dur": 16723.239,
    "args": {
      "External id": 2480465,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475880.952, "dur": 5.545,
    "args": {
      "External id": 2480466,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656475888.332, "dur": 16707.040,
    "args": {
      "External id": 2480467,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656492812.626, "dur": 34.993,
    "args": {
      "External id": 2480468,"Sequence number": 24740475, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8171
    }
  },
  {
    "ph": "s", "id": 173, "pid": 1336760, "tid": 1336760, "ts": 1514656492812.626,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656492835.621, "dur": 7.241,
    "args": {
      "External id": 2480469,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656492838.638, "dur": 3.995,
    "args": {
      "External id": 2480470,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656492910.830, "dur": 106.321,
    "args": {
      "External id": 2480471,"Record function id": 0, "Ev Idx": 8174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493020.794, "dur": 1108.544,
    "args": {
      "External id": 2480472,"Record function id": 0, "Ev Idx": 8175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493062.076, "dur": 1053.811,
    "args": {
      "External id": 2480473,"Sequence number": 24740476, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8176
    }
  },
  {
    "ph": "s", "id": 172, "pid": 1336760, "tid": 1336760, "ts": 1514656493062.076,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493133.460, "dur": 44.485,
    "args": {
      "External id": 2480474,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493192.044, "dur": 102.403,
    "args": {
      "External id": 2480475,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493304.004, "dur": 39.121,
    "args": {
      "External id": 2480476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493351.951, "dur": 31.771,
    "args": {
      "External id": 2480477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493407.157, "dur": 25.643,
    "args": {
      "External id": 2480478,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493452.387, "dur": 16.331,
    "args": {
      "External id": 2480479,"kernel_hash": "cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ld/cldqui4codyczhkdrz3ctt5pogllxs6yduwk43jmvbof3ofxmkjv.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493485.446, "dur": 150.366,
    "args": {
      "External id": 2480480,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493552.070, "dur": 12.658,
    "args": {
      "External id": 2480481,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493557.017, "dur": 6.798,
    "args": {
      "External id": 2480482,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493567.697, "dur": 5.816,
    "args": {
      "External id": 2480483,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493575.000, "dur": 2.086,
    "args": {
      "External id": 2480484,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493579.552, "dur": 2.894,
    "args": {
      "External id": 2480485,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493647.585, "dur": 52.202,
    "args": {
      "External id": 2480486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493734.227, "dur": 30.447,
    "args": {
      "External id": 2480487,"kernel_hash": "cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/of/cofo2xf3bigdfueamjfcjan5mqabwcl7bhbdhlz6jilhn2xeraxt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493772.773, "dur": 43.871,
    "args": {
      "External id": 2480488,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493825.802, "dur": 35.084,
    "args": {
      "External id": 2480489,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493882.835, "dur": 31.323,
    "args": {
      "External id": 2480490,"kernel_hash": "c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/4q/c4qey7euxloybjhoogrguqzxinnks6rjj7vhn5aetaz3osquskbk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493920.059, "dur": 33.999,
    "args": {
      "External id": 2480491,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336760, "tid": 1336760,
    "ts": 1514656493976.920, "dur": 66.216,
    "args": {
      "External id": 2480492,"kernel_hash": "cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/yz/cyz3pl7hmgkcq4azweuf2wmz7unjnnxmcmzlxe36c4wfauvm33mj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.27)", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494194.083, "dur": 32.798,
    "args": {
      "External id": 2480493,"Record function id": 0, "Ev Idx": 8196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494369.587, "dur": 321.735,
    "args": {
      "External id": 2480494,"Sequence number": 24740477, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8197
    }
  },
  {
    "ph": "s", "id": 171, "pid": 1336760, "tid": 1336760, "ts": 1514656494369.587,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494401.406, "dur": 7.750,
    "args": {
      "External id": 2480495,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494403.259, "dur": 5.604,
    "args": {
      "External id": 2480496,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494418.826, "dur": 13.234,
    "args": {
      "External id": 2480497,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494422.452, "dur": 8.943,
    "args": {
      "External id": 2480498,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494441.639, "dur": 5.058,
    "args": {
      "External id": 2480499,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494667.570, "dur": 9.665,
    "args": {
      "External id": 2480500,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494673.151, "dur": 3.764,
    "args": {
      "External id": 2480501,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494722.020, "dur": 134.577,
    "args": {
      "External id": 2480502,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494725.068, "dur": 14.985,
    "args": {
      "External id": 2480503,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8206
    }
  },
  {
    "ph": "s", "id": 170, "pid": 1336760, "tid": 1336760, "ts": 1514656494725.068,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494730.485, "dur": 7.663,
    "args": {
      "External id": 2480504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494735.671, "dur": 2.125,
    "args": {
      "External id": 2480505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494742.929, "dur": 113.240,
    "args": {
      "External id": 2480506,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494746.062, "dur": 5.232,
    "args": {
      "External id": 2480507,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494747.228, "dur": 3.899,
    "args": {
      "External id": 2480508,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8211
    }
  },
  {
    "ph": "s", "id": 169, "pid": 1336760, "tid": 1336760, "ts": 1514656494747.228,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494755.356, "dur": 92.369,
    "args": {
      "External id": 2480509,"Sequence number": 24740480, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8212
    }
  },
  {
    "ph": "s", "id": 168, "pid": 1336760, "tid": 1336760, "ts": 1514656494755.356,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494851.110, "dur": 4.014,
    "args": {
      "External id": 2480510,"Sequence number": 24740481, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8213
    }
  },
  {
    "ph": "s", "id": 167, "pid": 1336760, "tid": 1336760, "ts": 1514656494851.110,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494867.736, "dur": 71.125,
    "args": {
      "External id": 2480511,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494868.597, "dur": 8.847,
    "args": {
      "External id": 2480512,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8215
    }
  },
  {
    "ph": "s", "id": 166, "pid": 1336760, "tid": 1336760, "ts": 1514656494868.597,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494870.833, "dur": 5.324,
    "args": {
      "External id": 2480513,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494874.889, "dur": 1.055,
    "args": {
      "External id": 2480514,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494878.001, "dur": 60.585,
    "args": {
      "External id": 2480515,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494879.701, "dur": 5.808,
    "args": {
      "External id": 2480516,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494880.625, "dur": 4.716,
    "args": {
      "External id": 2480517,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8220
    }
  },
  {
    "ph": "s", "id": 165, "pid": 1336760, "tid": 1336760, "ts": 1514656494880.625,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494886.123, "dur": 45.528,
    "args": {
      "External id": 2480518,"Sequence number": 24740484, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8221
    }
  },
  {
    "ph": "s", "id": 164, "pid": 1336760, "tid": 1336760, "ts": 1514656494886.123,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494933.384, "dur": 4.797,
    "args": {
      "External id": 2480519,"Sequence number": 24740485, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8222
    }
  },
  {
    "ph": "s", "id": 163, "pid": 1336760, "tid": 1336760, "ts": 1514656494933.384,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494947.206, "dur": 113.942,
    "args": {
      "External id": 2480520,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494947.755, "dur": 5.215,
    "args": {
      "External id": 2480521,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8224
    }
  },
  {
    "ph": "s", "id": 162, "pid": 1336760, "tid": 1336760, "ts": 1514656494947.755,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494949.299, "dur": 2.529,
    "args": {
      "External id": 2480522,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494950.785, "dur": 0.876,
    "args": {
      "External id": 2480523,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494955.721, "dur": 105.189,
    "args": {
      "External id": 2480524,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494956.648, "dur": 5.077,
    "args": {
      "External id": 2480525,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494957.624, "dur": 3.977,
    "args": {
      "External id": 2480526,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8229
    }
  },
  {
    "ph": "s", "id": 161, "pid": 1336760, "tid": 1336760, "ts": 1514656494957.624,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656494962.169, "dur": 88.114,
    "args": {
      "External id": 2480527,"Sequence number": 24740488, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8230
    }
  },
  {
    "ph": "s", "id": 160, "pid": 1336760, "tid": 1336760, "ts": 1514656494962.169,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495053.879, "dur": 6.568,
    "args": {
      "External id": 2480528,"Sequence number": 24740489, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8231
    }
  },
  {
    "ph": "s", "id": 159, "pid": 1336760, "tid": 1336760, "ts": 1514656495053.879,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495084.997, "dur": 5.202,
    "args": {
      "External id": 2480529,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495086.452, "dur": 3.640,
    "args": {
      "External id": 2480530,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8233
    }
  },
  {
    "ph": "s", "id": 158, "pid": 1336760, "tid": 1336760, "ts": 1514656495086.452,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495096.832, "dur": 3.415,
    "args": {
      "External id": 2480531,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495098.204, "dur": 1.922,
    "args": {
      "External id": 2480532,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8235
    }
  },
  {
    "ph": "s", "id": 157, "pid": 1336760, "tid": 1336760, "ts": 1514656495098.204,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495106.302, "dur": 4.913,
    "args": {
      "External id": 2480533,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495107.558, "dur": 3.520,
    "args": {
      "External id": 2480534,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8237
    }
  },
  {
    "ph": "s", "id": 156, "pid": 1336760, "tid": 1336760, "ts": 1514656495107.558,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495146.289, "dur": 190.624,
    "args": {
      "External id": 2480535,"Sequence number": 24740493, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8238
    }
  },
  {
    "ph": "s", "id": 155, "pid": 1336760, "tid": 1336760, "ts": 1514656495146.289,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495170.082, "dur": 10.432,
    "args": {
      "External id": 2480536,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495173.535, "dur": 6.390,
    "args": {
      "External id": 2480537,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495350.217, "dur": 114.244,
    "args": {
      "External id": 2480538,"Sequence number": 24740494, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8241
    }
  },
  {
    "ph": "s", "id": 154, "pid": 1336760, "tid": 1336760, "ts": 1514656495350.217,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495365.174, "dur": 8.304,
    "args": {
      "External id": 2480539,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495367.978, "dur": 5.103,
    "args": {
      "External id": 2480540,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495494.259, "dur": 220.025,
    "args": {
      "External id": 2480541,"Sequence number": 24740495, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8244
    }
  },
  {
    "ph": "s", "id": 153, "pid": 1336760, "tid": 1336760, "ts": 1514656495494.259,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495525.848, "dur": 155.280,
    "args": {
      "External id": 2480542,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495597.058, "dur": 8.804,
    "args": {
      "External id": 2480543,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495599.764, "dur": 5.517,
    "args": {
      "External id": 2480544,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495608.588, "dur": 4.662,
    "args": {
      "External id": 2480545,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495614.648, "dur": 1.582,
    "args": {
      "External id": 2480546,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495620.966, "dur": 3.478,
    "args": {
      "External id": 2480547,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495697.284, "dur": 5.915,
    "args": {
      "External id": 2480548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495719.747, "dur": 6.485,
    "args": {
      "External id": 2480549,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495721.228, "dur": 4.821,
    "args": {
      "External id": 2480550,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8253
    }
  },
  {
    "ph": "s", "id": 152, "pid": 1336760, "tid": 1336760, "ts": 1514656495721.228,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495744.559, "dur": 128.093,
    "args": {
      "External id": 2480551,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495748.104, "dur": 10.035,
    "args": {
      "External id": 2480552,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8255
    }
  },
  {
    "ph": "s", "id": 151, "pid": 1336760, "tid": 1336760, "ts": 1514656495748.104,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495751.723, "dur": 4.835,
    "args": {
      "External id": 2480553,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495754.229, "dur": 2.010,
    "args": {
      "External id": 2480554,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495759.478, "dur": 112.807,
    "args": {
      "External id": 2480555,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495761.774, "dur": 9.398,
    "args": {
      "External id": 2480556,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495764.602, "dur": 6.378,
    "args": {
      "External id": 2480557,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8260
    }
  },
  {
    "ph": "s", "id": 150, "pid": 1336760, "tid": 1336760, "ts": 1514656495764.602,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495772.188, "dur": 91.001,
    "args": {
      "External id": 2480558,"Sequence number": 24740499, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8261
    }
  },
  {
    "ph": "s", "id": 149, "pid": 1336760, "tid": 1336760, "ts": 1514656495772.188,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495865.824, "dur": 5.661,
    "args": {
      "External id": 2480559,"Sequence number": 24740500, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8262
    }
  },
  {
    "ph": "s", "id": 148, "pid": 1336760, "tid": 1336760, "ts": 1514656495865.824,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495910.031, "dur": 276.660,
    "args": {
      "External id": 2480560,"Sequence number": 24740501, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8263
    }
  },
  {
    "ph": "s", "id": 147, "pid": 1336760, "tid": 1336760, "ts": 1514656495910.031,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495931.444, "dur": 3.106,
    "args": {
      "External id": 2480561,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495932.320, "dur": 2.066,
    "args": {
      "External id": 2480562,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495940.289, "dur": 3.842,
    "args": {
      "External id": 2480563,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495941.464, "dur": 2.540,
    "args": {
      "External id": 2480564,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495942.717, "dur": 1.189,
    "args": {
      "External id": 2480565,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495951.819, "dur": 8.943,
    "args": {
      "External id": 2480566,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495955.910, "dur": 4.506,
    "args": {
      "External id": 2480567,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495966.833, "dur": 3.680,
    "args": {
      "External id": 2480568,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656495974.329, "dur": 3.125,
    "args": {
      "External id": 2480569,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496159.204, "dur": 4.518,
    "args": {
      "External id": 2480570,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496160.491, "dur": 2.978,
    "args": {
      "External id": 2480571,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496166.401, "dur": 2.368,
    "args": {
      "External id": 2480572,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496167.636, "dur": 1.017,
    "args": {
      "External id": 2480573,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496211.317, "dur": 105.648,
    "args": {
      "External id": 2480574,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496212.691, "dur": 8.544,
    "args": {
      "External id": 2480575,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8278
    }
  },
  {
    "ph": "s", "id": 146, "pid": 1336760, "tid": 1336760, "ts": 1514656496212.691,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496215.626, "dur": 4.315,
    "args": {
      "External id": 2480576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496217.632, "dur": 1.780,
    "args": {
      "External id": 2480577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496222.331, "dur": 94.302,
    "args": {
      "External id": 2480578,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496225.601, "dur": 4.197,
    "args": {
      "External id": 2480579,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496226.473, "dur": 3.211,
    "args": {
      "External id": 2480580,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8283
    }
  },
  {
    "ph": "s", "id": 145, "pid": 1336760, "tid": 1336760, "ts": 1514656496226.473,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496230.361, "dur": 79.519,
    "args": {
      "External id": 2480581,"Sequence number": 24740504, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8284
    }
  },
  {
    "ph": "s", "id": 144, "pid": 1336760, "tid": 1336760, "ts": 1514656496230.361,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496312.062, "dur": 3.906,
    "args": {
      "External id": 2480582,"Sequence number": 24740505, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8285
    }
  },
  {
    "ph": "s", "id": 143, "pid": 1336760, "tid": 1336760, "ts": 1514656496312.062,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496326.543, "dur": 75.179,
    "args": {
      "External id": 2480583,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496329.583, "dur": 6.861,
    "args": {
      "External id": 2480584,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8287
    }
  },
  {
    "ph": "s", "id": 142, "pid": 1336760, "tid": 1336760, "ts": 1514656496329.583,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496332.052, "dur": 2.938,
    "args": {
      "External id": 2480585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496333.848, "dur": 0.993,
    "args": {
      "External id": 2480586,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496337.175, "dur": 64.325,
    "args": {
      "External id": 2480587,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496338.704, "dur": 9.398,
    "args": {
      "External id": 2480588,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496341.637, "dur": 6.317,
    "args": {
      "External id": 2480589,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8292
    }
  },
  {
    "ph": "s", "id": 141, "pid": 1336760, "tid": 1336760, "ts": 1514656496341.637,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496348.839, "dur": 45.889,
    "args": {
      "External id": 2480590,"Sequence number": 24740508, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8293
    }
  },
  {
    "ph": "s", "id": 140, "pid": 1336760, "tid": 1336760, "ts": 1514656496348.839,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496396.242, "dur": 4.913,
    "args": {
      "External id": 2480591,"Sequence number": 24740509, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8294
    }
  },
  {
    "ph": "s", "id": 139, "pid": 1336760, "tid": 1336760, "ts": 1514656496396.242,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496425.818, "dur": 186.119,
    "args": {
      "External id": 2480592,"Sequence number": 24740510, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8295
    }
  },
  {
    "ph": "s", "id": 138, "pid": 1336760, "tid": 1336760, "ts": 1514656496425.818,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496470.259, "dur": 4.501,
    "args": {
      "External id": 2480593,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496510.141, "dur": 85.384,
    "args": {
      "External id": 2480594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496510.874, "dur": 5.576,
    "args": {
      "External id": 2480595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496512.056, "dur": 3.480,
    "args": {
      "External id": 2480596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496514.062, "dur": 1.323,
    "args": {
      "External id": 2480597,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496517.265, "dur": 77.927,
    "args": {
      "External id": 2480598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496520.688, "dur": 3.041,
    "args": {
      "External id": 2480599,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496521.575, "dur": 2.003,
    "args": {
      "External id": 2480600,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496524.204, "dur": 66.000,
    "args": {
      "External id": 2480601,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496592.878, "dur": 1.401,
    "args": {
      "External id": 2480602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496621.737, "dur": 28.894,
    "args": {
      "External id": 2480603,"Sequence number": 24740511, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8306
    }
  },
  {
    "ph": "s", "id": 137, "pid": 1336760, "tid": 1336760, "ts": 1514656496621.737,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496688.363, "dur": 185.030,
    "args": {
      "External id": 2480604,"Sequence number": 24740512, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8307
    }
  },
  {
    "ph": "s", "id": 136, "pid": 1336760, "tid": 1336760, "ts": 1514656496688.363,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496709.010, "dur": 5.854,
    "args": {
      "External id": 2480605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496712.305, "dur": 2.430,
    "args": {
      "External id": 2480606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496722.977, "dur": 7.009,
    "args": {
      "External id": 2480607,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496725.492, "dur": 4.081,
    "args": {
      "External id": 2480608,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496735.588, "dur": 4.085,
    "args": {
      "External id": 2480609,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496858.159, "dur": 3.608,
    "args": {
      "External id": 2480610,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496859.800, "dur": 1.764,
    "args": {
      "External id": 2480611,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496894.045, "dur": 155.324,
    "args": {
      "External id": 2480612,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496895.430, "dur": 9.220,
    "args": {
      "External id": 2480613,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8316
    }
  },
  {
    "ph": "s", "id": 135, "pid": 1336760, "tid": 1336760, "ts": 1514656496895.430,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496899.930, "dur": 3.421,
    "args": {
      "External id": 2480614,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496901.993, "dur": 1.210,
    "args": {
      "External id": 2480615,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496905.743, "dur": 143.158,
    "args": {
      "External id": 2480616,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496907.356, "dur": 4.900,
    "args": {
      "External id": 2480617,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496909.404, "dur": 2.734,
    "args": {
      "External id": 2480618,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8321
    }
  },
  {
    "ph": "s", "id": 134, "pid": 1336760, "tid": 1336760, "ts": 1514656496909.404,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496913.077, "dur": 62.405,
    "args": {
      "External id": 2480619,"Sequence number": 24740515, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8322
    }
  },
  {
    "ph": "s", "id": 133, "pid": 1336760, "tid": 1336760, "ts": 1514656496913.077,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656496977.300, "dur": 70.157,
    "args": {
      "External id": 2480620,"Sequence number": 24740516, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8323
    }
  },
  {
    "ph": "s", "id": 132, "pid": 1336760, "tid": 1336760, "ts": 1514656496977.300,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497062.345, "dur": 83.376,
    "args": {
      "External id": 2480621,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497063.106, "dur": 12.962,
    "args": {
      "External id": 2480622,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8325
    }
  },
  {
    "ph": "s", "id": 131, "pid": 1336760, "tid": 1336760, "ts": 1514656497063.106,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497070.509, "dur": 3.951,
    "args": {
      "External id": 2480623,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497072.813, "dur": 1.486,
    "args": {
      "External id": 2480624,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497076.924, "dur": 68.597,
    "args": {
      "External id": 2480625,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497077.894, "dur": 6.145,
    "args": {
      "External id": 2480626,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497078.887, "dur": 4.999,
    "args": {
      "External id": 2480627,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8330
    }
  },
  {
    "ph": "s", "id": 130, "pid": 1336760, "tid": 1336760, "ts": 1514656497078.887,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497084.824, "dur": 56.260,
    "args": {
      "External id": 2480628,"Sequence number": 24740519, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8331
    }
  },
  {
    "ph": "s", "id": 129, "pid": 1336760, "tid": 1336760, "ts": 1514656497084.824,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497142.873, "dur": 2.234,
    "args": {
      "External id": 2480629,"Sequence number": 24740520, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8332
    }
  },
  {
    "ph": "s", "id": 128, "pid": 1336760, "tid": 1336760, "ts": 1514656497142.873,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497154.495, "dur": 66.307,
    "args": {
      "External id": 2480630,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497155.031, "dur": 10.255,
    "args": {
      "External id": 2480631,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8334
    }
  },
  {
    "ph": "s", "id": 127, "pid": 1336760, "tid": 1336760, "ts": 1514656497155.031,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497156.918, "dur": 7.192,
    "args": {
      "External id": 2480632,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497160.937, "dur": 2.944,
    "args": {
      "External id": 2480633,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497166.221, "dur": 54.365,
    "args": {
      "External id": 2480634,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497167.673, "dur": 5.661,
    "args": {
      "External id": 2480635,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497168.738, "dur": 4.112,
    "args": {
      "External id": 2480636,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8339
    }
  },
  {
    "ph": "s", "id": 126, "pid": 1336760, "tid": 1336760, "ts": 1514656497168.738,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497175.943, "dur": 38.300,
    "args": {
      "External id": 2480637,"Sequence number": 24740523, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8340
    }
  },
  {
    "ph": "s", "id": 125, "pid": 1336760, "tid": 1336760, "ts": 1514656497175.943,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497215.752, "dur": 4.496,
    "args": {
      "External id": 2480638,"Sequence number": 24740524, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8341
    }
  },
  {
    "ph": "s", "id": 124, "pid": 1336760, "tid": 1336760, "ts": 1514656497215.752,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497238.060, "dur": 4.287,
    "args": {
      "External id": 2480639,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497239.200, "dur": 2.967,
    "args": {
      "External id": 2480640,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8343
    }
  },
  {
    "ph": "s", "id": 123, "pid": 1336760, "tid": 1336760, "ts": 1514656497239.200,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497276.572, "dur": 7.072,
    "args": {
      "External id": 2480641,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497279.906, "dur": 3.619,
    "args": {
      "External id": 2480642,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8345
    }
  },
  {
    "ph": "s", "id": 122, "pid": 1336760, "tid": 1336760, "ts": 1514656497279.906,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497288.218, "dur": 5.002,
    "args": {
      "External id": 2480643,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497288.902, "dur": 4.153,
    "args": {
      "External id": 2480644,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8347
    }
  },
  {
    "ph": "s", "id": 121, "pid": 1336760, "tid": 1336760, "ts": 1514656497288.902,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497324.776, "dur": 161.003,
    "args": {
      "External id": 2480645,"Sequence number": 24740528, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8348
    }
  },
  {
    "ph": "s", "id": 120, "pid": 1336760, "tid": 1336760, "ts": 1514656497324.776,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497345.448, "dur": 8.580,
    "args": {
      "External id": 2480646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497348.227, "dur": 5.231,
    "args": {
      "External id": 2480647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497499.645, "dur": 126.224,
    "args": {
      "External id": 2480648,"Sequence number": 24740529, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8351
    }
  },
  {
    "ph": "s", "id": 119, "pid": 1336760, "tid": 1336760, "ts": 1514656497499.645,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497514.354, "dur": 6.920,
    "args": {
      "External id": 2480649,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497516.346, "dur": 4.574,
    "args": {
      "External id": 2480650,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497655.094, "dur": 183.294,
    "args": {
      "External id": 2480651,"Sequence number": 24740530, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8354
    }
  },
  {
    "ph": "s", "id": 118, "pid": 1336760, "tid": 1336760, "ts": 1514656497655.094,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497683.895, "dur": 128.370,
    "args": {
      "External id": 2480652,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497733.664, "dur": 9.218,
    "args": {
      "External id": 2480653,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497736.324, "dur": 6.018,
    "args": {
      "External id": 2480654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497745.826, "dur": 4.507,
    "args": {
      "External id": 2480655,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497751.728, "dur": 3.254,
    "args": {
      "External id": 2480656,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497757.272, "dur": 3.348,
    "args": {
      "External id": 2480657,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497822.959, "dur": 4.824,
    "args": {
      "External id": 2480658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497843.341, "dur": 7.650,
    "args": {
      "External id": 2480659,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497844.595, "dur": 6.203,
    "args": {
      "External id": 2480660,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8363
    }
  },
  {
    "ph": "s", "id": 117, "pid": 1336760, "tid": 1336760, "ts": 1514656497844.595,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497867.576, "dur": 115.029,
    "args": {
      "External id": 2480661,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497868.952, "dur": 9.976,
    "args": {
      "External id": 2480662,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8365
    }
  },
  {
    "ph": "s", "id": 116, "pid": 1336760, "tid": 1336760, "ts": 1514656497868.952,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497871.601, "dur": 6.038,
    "args": {
      "External id": 2480663,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497875.708, "dur": 1.677,
    "args": {
      "External id": 2480664,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497879.994, "dur": 102.214,
    "args": {
      "External id": 2480665,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497883.616, "dur": 5.327,
    "args": {
      "External id": 2480666,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497884.339, "dur": 4.431,
    "args": {
      "External id": 2480667,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8370
    }
  },
  {
    "ph": "s", "id": 115, "pid": 1336760, "tid": 1336760, "ts": 1514656497884.339,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497889.869, "dur": 83.828,
    "args": {
      "External id": 2480668,"Sequence number": 24740534, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8371
    }
  },
  {
    "ph": "s", "id": 114, "pid": 1336760, "tid": 1336760, "ts": 1514656497889.869,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656497976.147, "dur": 5.340,
    "args": {
      "External id": 2480669,"Sequence number": 24740535, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8372
    }
  },
  {
    "ph": "s", "id": 113, "pid": 1336760, "tid": 1336760, "ts": 1514656497976.147,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498057.553, "dur": 227.216,
    "args": {
      "External id": 2480670,"Sequence number": 24740536, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8373
    }
  },
  {
    "ph": "s", "id": 112, "pid": 1336760, "tid": 1336760, "ts": 1514656498057.553,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498082.092, "dur": 3.698,
    "args": {
      "External id": 2480671,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498082.989, "dur": 2.395,
    "args": {
      "External id": 2480672,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498089.567, "dur": 4.410,
    "args": {
      "External id": 2480673,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498090.589, "dur": 3.279,
    "args": {
      "External id": 2480674,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498092.937, "dur": 0.824,
    "args": {
      "External id": 2480675,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498101.928, "dur": 8.065,
    "args": {
      "External id": 2480676,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498104.144, "dur": 5.422,
    "args": {
      "External id": 2480677,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498116.185, "dur": 3.670,
    "args": {
      "External id": 2480678,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498123.318, "dur": 3.725,
    "args": {
      "External id": 2480679,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498260.095, "dur": 3.201,
    "args": {
      "External id": 2480680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498261.007, "dur": 2.105,
    "args": {
      "External id": 2480681,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498267.821, "dur": 1.723,
    "args": {
      "External id": 2480682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498268.522, "dur": 0.916,
    "args": {
      "External id": 2480683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498305.436, "dur": 102.320,
    "args": {
      "External id": 2480684,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498306.202, "dur": 9.025,
    "args": {
      "External id": 2480685,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8388
    }
  },
  {
    "ph": "s", "id": 111, "pid": 1336760, "tid": 1336760, "ts": 1514656498306.202,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498309.689, "dur": 4.186,
    "args": {
      "External id": 2480686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498311.602, "dur": 2.052,
    "args": {
      "External id": 2480687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498318.270, "dur": 89.157,
    "args": {
      "External id": 2480688,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498319.752, "dur": 2.949,
    "args": {
      "External id": 2480689,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498320.334, "dur": 2.215,
    "args": {
      "External id": 2480690,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8393
    }
  },
  {
    "ph": "s", "id": 110, "pid": 1336760, "tid": 1336760, "ts": 1514656498320.334,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498323.359, "dur": 76.306,
    "args": {
      "External id": 2480691,"Sequence number": 24740539, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8394
    }
  },
  {
    "ph": "s", "id": 109, "pid": 1336760, "tid": 1336760, "ts": 1514656498323.359,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498401.682, "dur": 4.970,
    "args": {
      "External id": 2480692,"Sequence number": 24740540, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8395
    }
  },
  {
    "ph": "s", "id": 108, "pid": 1336760, "tid": 1336760, "ts": 1514656498401.682,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498416.026, "dur": 71.095,
    "args": {
      "External id": 2480693,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498416.582, "dur": 7.071,
    "args": {
      "External id": 2480694,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8397
    }
  },
  {
    "ph": "s", "id": 107, "pid": 1336760, "tid": 1336760, "ts": 1514656498416.582,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498419.035, "dur": 3.322,
    "args": {
      "External id": 2480695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498420.945, "dur": 1.124,
    "args": {
      "External id": 2480696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498424.202, "dur": 62.715,
    "args": {
      "External id": 2480697,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498427.093, "dur": 6.742,
    "args": {
      "External id": 2480698,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498428.221, "dur": 5.432,
    "args": {
      "External id": 2480699,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8402
    }
  },
  {
    "ph": "s", "id": 106, "pid": 1336760, "tid": 1336760, "ts": 1514656498428.221,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498434.419, "dur": 45.237,
    "args": {
      "External id": 2480700,"Sequence number": 24740543, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8403
    }
  },
  {
    "ph": "s", "id": 105, "pid": 1336760, "tid": 1336760, "ts": 1514656498434.419,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498481.741, "dur": 4.925,
    "args": {
      "External id": 2480701,"Sequence number": 24740544, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8404
    }
  },
  {
    "ph": "s", "id": 104, "pid": 1336760, "tid": 1336760, "ts": 1514656498481.741,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498509.136, "dur": 174.600,
    "args": {
      "External id": 2480702,"Sequence number": 24740545, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8405
    }
  },
  {
    "ph": "s", "id": 103, "pid": 1336760, "tid": 1336760, "ts": 1514656498509.136,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498565.089, "dur": 5.355,
    "args": {
      "External id": 2480703,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498605.013, "dur": 64.440,
    "args": {
      "External id": 2480704,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498605.716, "dur": 5.735,
    "args": {
      "External id": 2480705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498606.998, "dur": 3.394,
    "args": {
      "External id": 2480706,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498608.756, "dur": 1.384,
    "args": {
      "External id": 2480707,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498614.345, "dur": 54.712,
    "args": {
      "External id": 2480708,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498615.579, "dur": 2.578,
    "args": {
      "External id": 2480709,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498616.192, "dur": 1.814,
    "args": {
      "External id": 2480710,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498618.599, "dur": 45.925,
    "args": {
      "External id": 2480711,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498666.865, "dur": 1.162,
    "args": {
      "External id": 2480712,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498691.996, "dur": 24.840,
    "args": {
      "External id": 2480713,"Sequence number": 24740546, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8416
    }
  },
  {
    "ph": "s", "id": 102, "pid": 1336760, "tid": 1336760, "ts": 1514656498691.996,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498753.179, "dur": 177.138,
    "args": {
      "External id": 2480714,"Sequence number": 24740547, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8417
    }
  },
  {
    "ph": "s", "id": 101, "pid": 1336760, "tid": 1336760, "ts": 1514656498753.179,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498772.393, "dur": 3.324,
    "args": {
      "External id": 2480715,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498773.537, "dur": 1.890,
    "args": {
      "External id": 2480716,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498783.398, "dur": 6.624,
    "args": {
      "External id": 2480717,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498785.842, "dur": 3.770,
    "args": {
      "External id": 2480718,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498795.385, "dur": 3.661,
    "args": {
      "External id": 2480719,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498914.305, "dur": 2.696,
    "args": {
      "External id": 2480720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498915.138, "dur": 1.697,
    "args": {
      "External id": 2480721,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498949.382, "dur": 126.728,
    "args": {
      "External id": 2480722,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498950.218, "dur": 6.398,
    "args": {
      "External id": 2480723,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8426
    }
  },
  {
    "ph": "s", "id": 100, "pid": 1336760, "tid": 1336760, "ts": 1514656498950.218,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498952.000, "dur": 3.493,
    "args": {
      "External id": 2480724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498954.145, "dur": 1.152,
    "args": {
      "External id": 2480725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498957.465, "dur": 118.324,
    "args": {
      "External id": 2480726,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498960.845, "dur": 4.093,
    "args": {
      "External id": 2480727,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498961.709, "dur": 3.116,
    "args": {
      "External id": 2480728,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8431
    }
  },
  {
    "ph": "s", "id": 99, "pid": 1336760, "tid": 1336760, "ts": 1514656498961.709,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656498965.636, "dur": 100.678,
    "args": {
      "External id": 2480729,"Sequence number": 24740550, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8432
    }
  },
  {
    "ph": "s", "id": 98, "pid": 1336760, "tid": 1336760, "ts": 1514656498965.636,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499069.724, "dur": 5.321,
    "args": {
      "External id": 2480730,"Sequence number": 24740551, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8433
    }
  },
  {
    "ph": "s", "id": 97, "pid": 1336760, "tid": 1336760, "ts": 1514656499069.724,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499086.853, "dur": 67.955,
    "args": {
      "External id": 2480731,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499089.764, "dur": 6.170,
    "args": {
      "External id": 2480732,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8435
    }
  },
  {
    "ph": "s", "id": 96, "pid": 1336760, "tid": 1336760, "ts": 1514656499089.764,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499091.635, "dur": 3.036,
    "args": {
      "External id": 2480733,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499093.363, "dur": 1.173,
    "args": {
      "External id": 2480734,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499096.668, "dur": 57.885,
    "args": {
      "External id": 2480735,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499097.993, "dur": 5.520,
    "args": {
      "External id": 2480736,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499100.729, "dur": 2.661,
    "args": {
      "External id": 2480737,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8440
    }
  },
  {
    "ph": "s", "id": 95, "pid": 1336760, "tid": 1336760, "ts": 1514656499100.729,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499103.998, "dur": 46.000,
    "args": {
      "External id": 2480738,"Sequence number": 24740554, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8441
    }
  },
  {
    "ph": "s", "id": 94, "pid": 1336760, "tid": 1336760, "ts": 1514656499103.998,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499151.686, "dur": 2.548,
    "args": {
      "External id": 2480739,"Sequence number": 24740555, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8442
    }
  },
  {
    "ph": "s", "id": 93, "pid": 1336760, "tid": 1336760, "ts": 1514656499151.686,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499161.974, "dur": 62.762,
    "args": {
      "External id": 2480740,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499162.706, "dur": 9.474,
    "args": {
      "External id": 2480741,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8444
    }
  },
  {
    "ph": "s", "id": 92, "pid": 1336760, "tid": 1336760, "ts": 1514656499162.706,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499166.138, "dur": 4.752,
    "args": {
      "External id": 2480742,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499167.941, "dur": 2.730,
    "args": {
      "External id": 2480743,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499172.827, "dur": 51.645,
    "args": {
      "External id": 2480744,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499173.976, "dur": 6.281,
    "args": {
      "External id": 2480745,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499176.591, "dur": 3.554,
    "args": {
      "External id": 2480746,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8449
    }
  },
  {
    "ph": "s", "id": 91, "pid": 1336760, "tid": 1336760, "ts": 1514656499176.591,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499180.917, "dur": 37.580,
    "args": {
      "External id": 2480747,"Sequence number": 24740558, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8450
    }
  },
  {
    "ph": "s", "id": 90, "pid": 1336760, "tid": 1336760, "ts": 1514656499180.917,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499220.056, "dur": 4.073,
    "args": {
      "External id": 2480748,"Sequence number": 24740559, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8451
    }
  },
  {
    "ph": "s", "id": 89, "pid": 1336760, "tid": 1336760, "ts": 1514656499220.056,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499240.701, "dur": 5.624,
    "args": {
      "External id": 2480749,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499241.499, "dur": 4.489,
    "args": {
      "External id": 2480750,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8453
    }
  },
  {
    "ph": "s", "id": 88, "pid": 1336760, "tid": 1336760, "ts": 1514656499241.499,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499252.896, "dur": 3.253,
    "args": {
      "External id": 2480751,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499254.255, "dur": 1.762,
    "args": {
      "External id": 2480752,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8455
    }
  },
  {
    "ph": "s", "id": 87, "pid": 1336760, "tid": 1336760, "ts": 1514656499254.255,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499260.408, "dur": 3.735,
    "args": {
      "External id": 2480753,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499261.701, "dur": 2.308,
    "args": {
      "External id": 2480754,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8457
    }
  },
  {
    "ph": "s", "id": 86, "pid": 1336760, "tid": 1336760, "ts": 1514656499261.701,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499293.039, "dur": 161.489,
    "args": {
      "External id": 2480755,"Sequence number": 24740563, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8458
    }
  },
  {
    "ph": "s", "id": 85, "pid": 1336760, "tid": 1336760, "ts": 1514656499293.039,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499314.740, "dur": 10.893,
    "args": {
      "External id": 2480756,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499317.936, "dur": 7.187,
    "args": {
      "External id": 2480757,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499466.859, "dur": 131.359,
    "args": {
      "External id": 2480758,"Sequence number": 24740564, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8461
    }
  },
  {
    "ph": "s", "id": 84, "pid": 1336760, "tid": 1336760, "ts": 1514656499466.859,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499479.999, "dur": 7.367,
    "args": {
      "External id": 2480759,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499482.356, "dur": 4.588,
    "args": {
      "External id": 2480760,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499630.338, "dur": 172.737,
    "args": {
      "External id": 2480761,"Sequence number": 24740565, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8464
    }
  },
  {
    "ph": "s", "id": 83, "pid": 1336760, "tid": 1336760, "ts": 1514656499630.338,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499657.629, "dur": 120.512,
    "args": {
      "External id": 2480762,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499703.050, "dur": 8.494,
    "args": {
      "External id": 2480763,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499705.812, "dur": 5.189,
    "args": {
      "External id": 2480764,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499714.278, "dur": 3.819,
    "args": {
      "External id": 2480765,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499719.024, "dur": 1.668,
    "args": {
      "External id": 2480766,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499723.272, "dur": 3.521,
    "args": {
      "External id": 2480767,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499789.342, "dur": 4.553,
    "args": {
      "External id": 2480768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499807.637, "dur": 5.684,
    "args": {
      "External id": 2480769,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499809.428, "dur": 3.741,
    "args": {
      "External id": 2480770,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8473
    }
  },
  {
    "ph": "s", "id": 82, "pid": 1336760, "tid": 1336760, "ts": 1514656499809.428,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499824.793, "dur": 111.336,
    "args": {
      "External id": 2480771,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499826.224, "dur": 11.820,
    "args": {
      "External id": 2480772,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8475
    }
  },
  {
    "ph": "s", "id": 81, "pid": 1336760, "tid": 1336760, "ts": 1514656499826.224,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499831.100, "dur": 5.742,
    "args": {
      "External id": 2480773,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499835.099, "dur": 1.502,
    "args": {
      "External id": 2480774,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499839.292, "dur": 96.412,
    "args": {
      "External id": 2480775,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499841.472, "dur": 3.253,
    "args": {
      "External id": 2480776,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499842.378, "dur": 2.190,
    "args": {
      "External id": 2480777,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8480
    }
  },
  {
    "ph": "s", "id": 80, "pid": 1336760, "tid": 1336760, "ts": 1514656499842.378,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499845.748, "dur": 81.175,
    "args": {
      "External id": 2480778,"Sequence number": 24740569, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8481
    }
  },
  {
    "ph": "s", "id": 79, "pid": 1336760, "tid": 1336760, "ts": 1514656499845.748,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499929.136, "dur": 5.927,
    "args": {
      "External id": 2480779,"Sequence number": 24740570, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8482
    }
  },
  {
    "ph": "s", "id": 78, "pid": 1336760, "tid": 1336760, "ts": 1514656499929.136,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499967.180, "dur": 248.342,
    "args": {
      "External id": 2480780,"Sequence number": 24740571, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8483
    }
  },
  {
    "ph": "s", "id": 77, "pid": 1336760, "tid": 1336760, "ts": 1514656499967.180,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656499983.700, "dur": 38.327,
    "args": {
      "External id": 2480781,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500018.823, "dur": 2.806,
    "args": {
      "External id": 2480782,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500027.735, "dur": 5.768,
    "args": {
      "External id": 2480783,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500031.510, "dur": 1.884,
    "args": {
      "External id": 2480784,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500032.274, "dur": 1.009,
    "args": {
      "External id": 2480785,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500040.724, "dur": 8.258,
    "args": {
      "External id": 2480786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500043.223, "dur": 5.366,
    "args": {
      "External id": 2480787,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500055.768, "dur": 4.094,
    "args": {
      "External id": 2480788,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500063.380, "dur": 4.070,
    "args": {
      "External id": 2480789,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500192.892, "dur": 3.126,
    "args": {
      "External id": 2480790,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500193.735, "dur": 2.080,
    "args": {
      "External id": 2480791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500198.449, "dur": 2.208,
    "args": {
      "External id": 2480792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500199.478, "dur": 1.075,
    "args": {
      "External id": 2480793,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500234.179, "dur": 110.370,
    "args": {
      "External id": 2480794,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500243.689, "dur": 9.845,
    "args": {
      "External id": 2480795,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8498
    }
  },
  {
    "ph": "s", "id": 76, "pid": 1336760, "tid": 1336760, "ts": 1514656500243.689,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500248.357, "dur": 3.649,
    "args": {
      "External id": 2480796,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500250.472, "dur": 1.314,
    "args": {
      "External id": 2480797,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500254.361, "dur": 89.873,
    "args": {
      "External id": 2480798,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500255.809, "dur": 7.404,
    "args": {
      "External id": 2480799,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500256.710, "dur": 6.304,
    "args": {
      "External id": 2480800,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8503
    }
  },
  {
    "ph": "s", "id": 75, "pid": 1336760, "tid": 1336760, "ts": 1514656500256.710,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500264.022, "dur": 69.963,
    "args": {
      "External id": 2480801,"Sequence number": 24740574, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8504
    }
  },
  {
    "ph": "s", "id": 74, "pid": 1336760, "tid": 1336760, "ts": 1514656500264.022,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500335.881, "dur": 7.712,
    "args": {
      "External id": 2480802,"Sequence number": 24740575, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8505
    }
  },
  {
    "ph": "s", "id": 73, "pid": 1336760, "tid": 1336760, "ts": 1514656500335.881,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500353.773, "dur": 71.029,
    "args": {
      "External id": 2480803,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500354.469, "dur": 8.300,
    "args": {
      "External id": 2480804,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8507
    }
  },
  {
    "ph": "s", "id": 72, "pid": 1336760, "tid": 1336760, "ts": 1514656500354.469,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500356.404, "dur": 5.011,
    "args": {
      "External id": 2480805,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500360.112, "dur": 0.923,
    "args": {
      "External id": 2480806,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500363.683, "dur": 60.879,
    "args": {
      "External id": 2480807,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500364.822, "dur": 4.138,
    "args": {
      "External id": 2480808,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500365.543, "dur": 3.257,
    "args": {
      "External id": 2480809,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8512
    }
  },
  {
    "ph": "s", "id": 71, "pid": 1336760, "tid": 1336760, "ts": 1514656500365.543,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500372.085, "dur": 47.870,
    "args": {
      "External id": 2480810,"Sequence number": 24740578, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8513
    }
  },
  {
    "ph": "s", "id": 70, "pid": 1336760, "tid": 1336760, "ts": 1514656500372.085,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500421.559, "dur": 2.691,
    "args": {
      "External id": 2480811,"Sequence number": 24740579, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8514
    }
  },
  {
    "ph": "s", "id": 69, "pid": 1336760, "tid": 1336760, "ts": 1514656500421.559,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500444.024, "dur": 187.817,
    "args": {
      "External id": 2480812,"Sequence number": 24740580, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8515
    }
  },
  {
    "ph": "s", "id": 68, "pid": 1336760, "tid": 1336760, "ts": 1514656500444.024,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500479.198, "dur": 3.849,
    "args": {
      "External id": 2480813,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500516.531, "dur": 100.177,
    "args": {
      "External id": 2480814,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500517.204, "dur": 4.635,
    "args": {
      "External id": 2480815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500518.096, "dur": 3.008,
    "args": {
      "External id": 2480816,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500520.026, "dur": 0.805,
    "args": {
      "External id": 2480817,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500522.385, "dur": 94.049,
    "args": {
      "External id": 2480818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500523.454, "dur": 2.620,
    "args": {
      "External id": 2480819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500524.426, "dur": 1.386,
    "args": {
      "External id": 2480820,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500544.257, "dur": 67.491,
    "args": {
      "External id": 2480821,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500614.301, "dur": 1.357,
    "args": {
      "External id": 2480822,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500640.804, "dur": 25.895,
    "args": {
      "External id": 2480823,"Sequence number": 24740581, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8526
    }
  },
  {
    "ph": "s", "id": 67, "pid": 1336760, "tid": 1336760, "ts": 1514656500640.804,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500704.176, "dur": 197.298,
    "args": {
      "External id": 2480824,"Sequence number": 24740582, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8527
    }
  },
  {
    "ph": "s", "id": 66, "pid": 1336760, "tid": 1336760, "ts": 1514656500704.176,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500721.569, "dur": 3.002,
    "args": {
      "External id": 2480825,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500722.341, "dur": 2.065,
    "args": {
      "External id": 2480826,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500735.538, "dur": 15.134,
    "args": {
      "External id": 2480827,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500745.795, "dur": 4.459,
    "args": {
      "External id": 2480828,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500761.016, "dur": 6.191,
    "args": {
      "External id": 2480829,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500889.341, "dur": 3.335,
    "args": {
      "External id": 2480830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500890.312, "dur": 2.175,
    "args": {
      "External id": 2480831,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500922.010, "dur": 133.463,
    "args": {
      "External id": 2480832,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500922.944, "dur": 9.961,
    "args": {
      "External id": 2480833,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8536
    }
  },
  {
    "ph": "s", "id": 65, "pid": 1336760, "tid": 1336760, "ts": 1514656500922.944,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500925.531, "dur": 6.188,
    "args": {
      "External id": 2480834,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500929.814, "dur": 1.717,
    "args": {
      "External id": 2480835,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500933.924, "dur": 121.235,
    "args": {
      "External id": 2480836,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500935.605, "dur": 3.327,
    "args": {
      "External id": 2480837,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500936.253, "dur": 2.556,
    "args": {
      "External id": 2480838,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8541
    }
  },
  {
    "ph": "s", "id": 64, "pid": 1336760, "tid": 1336760, "ts": 1514656500936.253,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656500941.835, "dur": 105.363,
    "args": {
      "External id": 2480839,"Sequence number": 24740585, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8542
    }
  },
  {
    "ph": "s", "id": 63, "pid": 1336760, "tid": 1336760, "ts": 1514656500941.835,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501050.447, "dur": 3.928,
    "args": {
      "External id": 2480840,"Sequence number": 24740586, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8543
    }
  },
  {
    "ph": "s", "id": 62, "pid": 1336760, "tid": 1336760, "ts": 1514656501050.447,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501066.560, "dur": 77.535,
    "args": {
      "External id": 2480841,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501067.299, "dur": 10.972,
    "args": {
      "External id": 2480842,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8545
    }
  },
  {
    "ph": "s", "id": 61, "pid": 1336760, "tid": 1336760, "ts": 1514656501067.299,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501069.343, "dur": 7.623,
    "args": {
      "External id": 2480843,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501073.461, "dur": 3.120,
    "args": {
      "External id": 2480844,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501079.120, "dur": 64.744,
    "args": {
      "External id": 2480845,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501080.063, "dur": 4.056,
    "args": {
      "External id": 2480846,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501081.064, "dur": 2.907,
    "args": {
      "External id": 2480847,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8550
    }
  },
  {
    "ph": "s", "id": 60, "pid": 1336760, "tid": 1336760, "ts": 1514656501081.064,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501084.694, "dur": 50.335,
    "args": {
      "External id": 2480848,"Sequence number": 24740589, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8551
    }
  },
  {
    "ph": "s", "id": 59, "pid": 1336760, "tid": 1336760, "ts": 1514656501084.694,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501136.755, "dur": 6.782,
    "args": {
      "External id": 2480849,"Sequence number": 24740590, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8552
    }
  },
  {
    "ph": "s", "id": 58, "pid": 1336760, "tid": 1336760, "ts": 1514656501136.755,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501152.366, "dur": 60.348,
    "args": {
      "External id": 2480850,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501153.032, "dur": 5.062,
    "args": {
      "External id": 2480851,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8554
    }
  },
  {
    "ph": "s", "id": 57, "pid": 1336760, "tid": 1336760, "ts": 1514656501153.032,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501154.581, "dur": 2.406,
    "args": {
      "External id": 2480852,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501155.992, "dur": 0.668,
    "args": {
      "External id": 2480853,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501160.841, "dur": 51.590,
    "args": {
      "External id": 2480854,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501161.814, "dur": 4.053,
    "args": {
      "External id": 2480855,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501162.457, "dur": 3.290,
    "args": {
      "External id": 2480856,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8559
    }
  },
  {
    "ph": "s", "id": 56, "pid": 1336760, "tid": 1336760, "ts": 1514656501162.457,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501166.293, "dur": 40.703,
    "args": {
      "External id": 2480857,"Sequence number": 24740593, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8560
    }
  },
  {
    "ph": "s", "id": 55, "pid": 1336760, "tid": 1336760, "ts": 1514656501166.293,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501208.435, "dur": 3.717,
    "args": {
      "External id": 2480858,"Sequence number": 24740594, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8561
    }
  },
  {
    "ph": "s", "id": 54, "pid": 1336760, "tid": 1336760, "ts": 1514656501208.435,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501228.964, "dur": 3.600,
    "args": {
      "External id": 2480859,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501229.446, "dur": 3.000,
    "args": {
      "External id": 2480860,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8563
    }
  },
  {
    "ph": "s", "id": 53, "pid": 1336760, "tid": 1336760, "ts": 1514656501229.446,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501239.488, "dur": 3.086,
    "args": {
      "External id": 2480861,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501240.730, "dur": 1.728,
    "args": {
      "External id": 2480862,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8565
    }
  },
  {
    "ph": "s", "id": 52, "pid": 1336760, "tid": 1336760, "ts": 1514656501240.730,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501249.191, "dur": 5.635,
    "args": {
      "External id": 2480863,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501250.563, "dur": 4.089,
    "args": {
      "External id": 2480864,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8567
    }
  },
  {
    "ph": "s", "id": 51, "pid": 1336760, "tid": 1336760, "ts": 1514656501250.563,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501285.273, "dur": 157.557,
    "args": {
      "External id": 2480865,"Sequence number": 24740598, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8568
    }
  },
  {
    "ph": "s", "id": 50, "pid": 1336760, "tid": 1336760, "ts": 1514656501285.273,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501304.338, "dur": 7.990,
    "args": {
      "External id": 2480866,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501306.980, "dur": 4.934,
    "args": {
      "External id": 2480867,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501455.349, "dur": 125.518,
    "args": {
      "External id": 2480868,"Sequence number": 24740599, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8571
    }
  },
  {
    "ph": "s", "id": 49, "pid": 1336760, "tid": 1336760, "ts": 1514656501455.349,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501468.821, "dur": 6.615,
    "args": {
      "External id": 2480869,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501470.802, "dur": 4.307,
    "args": {
      "External id": 2480870,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501609.315, "dur": 181.828,
    "args": {
      "External id": 2480871,"Sequence number": 24740600, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8574
    }
  },
  {
    "ph": "s", "id": 48, "pid": 1336760, "tid": 1336760, "ts": 1514656501609.315,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501635.952, "dur": 130.380,
    "args": {
      "External id": 2480872,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501688.005, "dur": 8.716,
    "args": {
      "External id": 2480873,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501690.246, "dur": 5.945,
    "args": {
      "External id": 2480874,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501699.416, "dur": 4.457,
    "args": {
      "External id": 2480875,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501705.294, "dur": 3.429,
    "args": {
      "External id": 2480876,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501711.089, "dur": 3.370,
    "args": {
      "External id": 2480877,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501777.597, "dur": 4.343,
    "args": {
      "External id": 2480878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501795.796, "dur": 6.458,
    "args": {
      "External id": 2480879,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501797.723, "dur": 4.409,
    "args": {
      "External id": 2480880,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8583
    }
  },
  {
    "ph": "s", "id": 47, "pid": 1336760, "tid": 1336760, "ts": 1514656501797.723,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501815.200, "dur": 104.921,
    "args": {
      "External id": 2480881,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501818.540, "dur": 8.151,
    "args": {
      "External id": 2480882,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8585
    }
  },
  {
    "ph": "s", "id": 46, "pid": 1336760, "tid": 1336760, "ts": 1514656501818.540,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501821.234, "dur": 4.368,
    "args": {
      "External id": 2480883,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501823.582, "dur": 1.733,
    "args": {
      "External id": 2480884,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501827.889, "dur": 91.971,
    "args": {
      "External id": 2480885,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501829.676, "dur": 6.032,
    "args": {
      "External id": 2480886,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501832.607, "dur": 2.964,
    "args": {
      "External id": 2480887,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8590
    }
  },
  {
    "ph": "s", "id": 45, "pid": 1336760, "tid": 1336760, "ts": 1514656501832.607,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501836.820, "dur": 76.326,
    "args": {
      "External id": 2480888,"Sequence number": 24740604, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8591
    }
  },
  {
    "ph": "s", "id": 44, "pid": 1336760, "tid": 1336760, "ts": 1514656501836.820,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501915.463, "dur": 3.730,
    "args": {
      "External id": 2480889,"Sequence number": 24740605, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8592
    }
  },
  {
    "ph": "s", "id": 43, "pid": 1336760, "tid": 1336760, "ts": 1514656501915.463,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501951.842, "dur": 247.881,
    "args": {
      "External id": 2480890,"Sequence number": 24740606, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8593
    }
  },
  {
    "ph": "s", "id": 42, "pid": 1336760, "tid": 1336760, "ts": 1514656501951.842,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501970.947, "dur": 2.841,
    "args": {
      "External id": 2480891,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501972.147, "dur": 1.456,
    "args": {
      "External id": 2480892,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501977.702, "dur": 3.296,
    "args": {
      "External id": 2480893,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501978.848, "dur": 2.041,
    "args": {
      "External id": 2480894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656501979.402, "dur": 1.390,
    "args": {
      "External id": 2480895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502023.734, "dur": 10.190,
    "args": {
      "External id": 2480896,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502027.327, "dur": 5.971,
    "args": {
      "External id": 2480897,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502041.078, "dur": 3.683,
    "args": {
      "External id": 2480898,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502048.383, "dur": 4.677,
    "args": {
      "External id": 2480899,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502177.001, "dur": 3.713,
    "args": {
      "External id": 2480900,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502177.984, "dur": 2.344,
    "args": {
      "External id": 2480901,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502183.305, "dur": 2.307,
    "args": {
      "External id": 2480902,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502184.300, "dur": 1.206,
    "args": {
      "External id": 2480903,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502220.369, "dur": 100.628,
    "args": {
      "External id": 2480904,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502221.548, "dur": 10.114,
    "args": {
      "External id": 2480905,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8608
    }
  },
  {
    "ph": "s", "id": 41, "pid": 1336760, "tid": 1336760, "ts": 1514656502221.548,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502223.884, "dur": 6.425,
    "args": {
      "External id": 2480906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502226.033, "dur": 3.881,
    "args": {
      "External id": 2480907,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502232.594, "dur": 88.058,
    "args": {
      "External id": 2480908,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502234.606, "dur": 5.997,
    "args": {
      "External id": 2480909,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502237.215, "dur": 3.268,
    "args": {
      "External id": 2480910,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8613
    }
  },
  {
    "ph": "s", "id": 40, "pid": 1336760, "tid": 1336760, "ts": 1514656502237.215,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502241.330, "dur": 71.573,
    "args": {
      "External id": 2480911,"Sequence number": 24740609, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8614
    }
  },
  {
    "ph": "s", "id": 39, "pid": 1336760, "tid": 1336760, "ts": 1514656502241.330,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502314.894, "dur": 5.127,
    "args": {
      "External id": 2480912,"Sequence number": 24740610, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8615
    }
  },
  {
    "ph": "s", "id": 38, "pid": 1336760, "tid": 1336760, "ts": 1514656502314.894,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502329.715, "dur": 72.984,
    "args": {
      "External id": 2480913,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502330.354, "dur": 9.635,
    "args": {
      "External id": 2480914,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8617
    }
  },
  {
    "ph": "s", "id": 37, "pid": 1336760, "tid": 1336760, "ts": 1514656502330.354,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502335.726, "dur": 3.021,
    "args": {
      "External id": 2480915,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502337.370, "dur": 1.178,
    "args": {
      "External id": 2480916,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502340.696, "dur": 61.822,
    "args": {
      "External id": 2480917,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502341.593, "dur": 7.559,
    "args": {
      "External id": 2480918,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502344.603, "dur": 4.420,
    "args": {
      "External id": 2480919,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8622
    }
  },
  {
    "ph": "s", "id": 36, "pid": 1336760, "tid": 1336760, "ts": 1514656502344.603,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502349.739, "dur": 47.801,
    "args": {
      "External id": 2480920,"Sequence number": 24740613, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8623
    }
  },
  {
    "ph": "s", "id": 35, "pid": 1336760, "tid": 1336760, "ts": 1514656502349.739,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502399.245, "dur": 2.850,
    "args": {
      "External id": 2480921,"Sequence number": 24740614, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8624
    }
  },
  {
    "ph": "s", "id": 34, "pid": 1336760, "tid": 1336760, "ts": 1514656502399.245,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502421.985, "dur": 169.375,
    "args": {
      "External id": 2480922,"Sequence number": 24740615, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8625
    }
  },
  {
    "ph": "s", "id": 33, "pid": 1336760, "tid": 1336760, "ts": 1514656502421.985,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502460.807, "dur": 4.029,
    "args": {
      "External id": 2480923,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502495.833, "dur": 82.325,
    "args": {
      "External id": 2480924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502496.577, "dur": 6.440,
    "args": {
      "External id": 2480925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502497.844, "dur": 4.271,
    "args": {
      "External id": 2480926,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502499.258, "dur": 2.664,
    "args": {
      "External id": 2480927,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502503.707, "dur": 73.904,
    "args": {
      "External id": 2480928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502505.067, "dur": 4.633,
    "args": {
      "External id": 2480929,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502507.990, "dur": 1.594,
    "args": {
      "External id": 2480930,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502510.458, "dur": 61.484,
    "args": {
      "External id": 2480931,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502575.321, "dur": 1.488,
    "args": {
      "External id": 2480932,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502600.056, "dur": 25.680,
    "args": {
      "External id": 2480933,"Sequence number": 24740616, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8636
    }
  },
  {
    "ph": "s", "id": 32, "pid": 1336760, "tid": 1336760, "ts": 1514656502600.056,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502644.108, "dur": 43.708,
    "args": {
      "External id": 2480934,"Sequence number": 24740617, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "-2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8637
    }
  },
  {
    "ph": "s", "id": 31, "pid": 1336760, "tid": 1336760, "ts": 1514656502644.108,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502653.909, "dur": 29.481,
    "args": {
      "External id": 2480935,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502684.940, "dur": 1.117,
    "args": {
      "External id": 2480936,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 8192], []], "Ev Idx": 8639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502726.243, "dur": 47.199,
    "args": {
      "External id": 2480937,"Record function id": 0, "Ev Idx": 8640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 2/0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502774.563, "dur": 207.379,
    "args": {
      "External id": 2480938,"Record function id": 0, "Ev Idx": 8641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502815.133, "dur": 158.058,
    "args": {
      "External id": 2480939,"Sequence number": 24740618, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1], [33554432, 8192, 2048, 1]], "Input Dims": [[2048], [16, 4096, 4, 2048]], "Ev Idx": 8642
    }
  },
  {
    "ph": "s", "id": 30, "pid": 1336760, "tid": 1336760, "ts": 1514656502815.133,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336760, "tid": 1336760,
    "ts": 1514656502889.447, "dur": 43.079,
    "args": {
      "External id": 2480940,"kernel_hash": "cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/qp/cqpt7mwgy5yqmh72goig5li3mr2l2sinhq27eouv3cyvn3ym4dh5.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [262144, 2048], [2048], [262144], [], [], [], [], [], [], [], [], []], "Ev Idx": 8643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503127.080, "dur": 43.236,
    "args": {
      "External id": 2480941,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503129.609, "dur": 6.743,
    "args": {
      "External id": 2480942,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503139.076, "dur": 30.962,
    "args": {
      "External id": 2480943,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503141.916, "dur": 27.506,
    "args": {
      "External id": 2480944,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503175.334, "dur": 21.226,
    "args": {
      "External id": 2480945,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503176.451, "dur": 3.068,
    "args": {
      "External id": 2480946,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503182.828, "dur": 13.477,
    "args": {
      "External id": 2480947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503183.445, "dur": 12.483,
    "args": {
      "External id": 2480948,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503199.161, "dur": 18.093,
    "args": {
      "External id": 2480949,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503200.364, "dur": 2.894,
    "args": {
      "External id": 2480950,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503203.801, "dur": 13.209,
    "args": {
      "External id": 2480951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503204.546, "dur": 12.108,
    "args": {
      "External id": 2480952,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503228.138, "dur": 0.760,
    "args": {
      "External id": 2480953,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 8656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503237.179, "dur": 11.975,
    "args": {
      "External id": 2480954,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "5", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503244.987, "dur": 2.428,
    "args": {
      "External id": 2480955,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503255.433, "dur": 7.003,
    "args": {
      "External id": 2480956,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503259.675, "dur": 0.823,
    "args": {
      "External id": 2480957,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503263.495, "dur": 3.568,
    "args": {
      "External id": 2480958,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503265.588, "dur": 0.741,
    "args": {
      "External id": 2480959,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503268.290, "dur": 4.087,
    "args": {
      "External id": 2480960,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503270.876, "dur": 0.960,
    "args": {
      "External id": 2480961,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503275.259, "dur": 3.120,
    "args": {
      "External id": 2480962,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503277.212, "dur": 0.638,
    "args": {
      "External id": 2480963,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503279.244, "dur": 3.491,
    "args": {
      "External id": 2480964,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503281.173, "dur": 0.873,
    "args": {
      "External id": 2480965,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503283.935, "dur": 3.055,
    "args": {
      "External id": 2480966,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4096, 4], [], [], [], []], "Ev Idx": 8669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503285.613, "dur": 0.877,
    "args": {
      "External id": 2480967,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503292.227, "dur": 5.915,
    "args": {
      "External id": 2480968,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4096, 4], [], []], "Ev Idx": 8671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503295.792, "dur": 1.567,
    "args": {
      "External id": 2480969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503302.048, "dur": 3.414,
    "args": {
      "External id": 2480970,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 8673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503303.938, "dur": 0.981,
    "args": {
      "External id": 2480971,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503310.933, "dur": 6.952,
    "args": {
      "External id": 2480972,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 8675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503315.998, "dur": 0.865,
    "args": {
      "External id": 2480973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503319.226, "dur": 2.776,
    "args": {
      "External id": 2480974,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 8677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503320.738, "dur": 0.764,
    "args": {
      "External id": 2480975,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 8678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503324.497, "dur": 8.029,
    "args": {
      "External id": 2480976,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8679
    }
  },
  {
    "ph": "s", "id": 29, "pid": 1336760, "tid": 1336760, "ts": 1514656503324.497,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503330.239, "dur": 0.757,
    "args": {
      "External id": 2480977,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503333.580, "dur": 4.703,
    "args": {
      "External id": 2480978,"Sequence number": 24740620, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8681
    }
  },
  {
    "ph": "s", "id": 28, "pid": 1336760, "tid": 1336760, "ts": 1514656503333.580,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503337.043, "dur": 0.510,
    "args": {
      "External id": 2480979,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503339.538, "dur": 6.585,
    "args": {
      "External id": 2480980,"Sequence number": 24740621, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 8683
    }
  },
  {
    "ph": "s", "id": 27, "pid": 1336760, "tid": 1336760, "ts": 1514656503339.538,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503344.567, "dur": 0.716,
    "args": {
      "External id": 2480981,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503347.464, "dur": 6.786,
    "args": {
      "External id": 2480982,"Sequence number": 24740622, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 8685
    }
  },
  {
    "ph": "s", "id": 26, "pid": 1336760, "tid": 1336760, "ts": 1514656503347.464,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503350.572, "dur": 2.902,
    "args": {
      "External id": 2480983,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 8686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503358.913, "dur": 38.088,
    "args": {
      "External id": 2480984,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503361.427, "dur": 35.371,
    "args": {
      "External id": 2480985,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503363.827, "dur": 6.306,
    "args": {
      "External id": 2480986,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 8689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503365.899, "dur": 3.672,
    "args": {
      "External id": 2480987,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503371.434, "dur": 24.703,
    "args": {
      "External id": 2480988,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 8691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503422.910, "dur": 4.258,
    "args": {
      "External id": 2480989,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8692
    }
  },
  {
    "ph": "s", "id": 25, "pid": 1336760, "tid": 1336760, "ts": 1514656503422.910,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503429.581, "dur": 1.402,
    "args": {
      "External id": 2480990,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503459.778, "dur": 70572.242,
    "args": {
      "External id": 2480991,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 8694
    }
  },
  {
    "ph": "s", "id": 24, "pid": 1336760, "tid": 1336760, "ts": 1514656503459.778,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503474.863, "dur": 30.803,
    "args": {
      "External id": 2480992,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503475.885, "dur": 29.558,
    "args": {
      "External id": 2480993,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503477.254, "dur": 7.200,
    "args": {
      "External id": 2480994,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503478.342, "dur": 5.623,
    "args": {
      "External id": 2480995,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503485.258, "dur": 19.698,
    "args": {
      "External id": 2480996,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 8699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503522.800, "dur": 45.157,
    "args": {
      "External id": 2480997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503523.747, "dur": 22.504,
    "args": {
      "External id": 2480998,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503526.059, "dur": 19.489,
    "args": {
      "External id": 2480999,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503547.675, "dur": 20.088,
    "args": {
      "External id": 2481000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 8703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503548.935, "dur": 18.448,
    "args": {
      "External id": 2481001,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503572.609, "dur": 31.203,
    "args": {
      "External id": 2481002,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503573.841, "dur": 7.078,
    "args": {
      "External id": 2481003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503576.916, "dur": 3.740,
    "args": {
      "External id": 2481004,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503589.944, "dur": 13.708,
    "args": {
      "External id": 2481005,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503590.450, "dur": 12.839,
    "args": {
      "External id": 2481006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 8709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503613.483, "dur": 20.490,
    "args": {
      "External id": 2481007,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503614.572, "dur": 5.472,
    "args": {
      "External id": 2481008,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503620.908, "dur": 12.789,
    "args": {
      "External id": 2481009,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 8712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503621.459, "dur": 11.933,
    "args": {
      "External id": 2481010,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503642.844, "dur": 28.664,
    "args": {
      "External id": 2481011,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503676.169, "dur": 63.935,
    "args": {
      "External id": 2481012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503679.777, "dur": 59.911,
    "args": {
      "External id": 2481013,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503685.821, "dur": 1.483,
    "args": {
      "External id": 2481014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 8717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503688.561, "dur": 29.529,
    "args": {
      "External id": 2481015,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503690.115, "dur": 27.726,
    "args": {
      "External id": 2481016,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 8719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503694.971, "dur": 3.144,
    "args": {
      "External id": 2481017,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503699.008, "dur": 18.291,
    "args": {
      "External id": 2481018,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 8721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503746.001, "dur": 64136.672,
    "args": {
      "External id": 2481019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514656503747.912, "dur": 64133.503,
    "args": {
      "External id": 2481020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567894.830, "dur": 8.215,
    "args": {
      "External id": 2481021,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567899.734, "dur": 1.156,
    "args": {
      "External id": 2481022,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567908.381, "dur": 148.823,
    "args": {
      "External id": 2481023,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567912.770, "dur": 9.665,
    "args": {
      "External id": 2481024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567915.060, "dur": 6.551,
    "args": {
      "External id": 2481025,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567920.394, "dur": 0.936,
    "args": {
      "External id": 2481026,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567923.894, "dur": 132.305,
    "args": {
      "External id": 2481027,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656567926.086, "dur": 128.714,
    "args": {
      "External id": 2481028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568062.624, "dur": 5.763,
    "args": {
      "External id": 2481029,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568065.785, "dur": 0.721,
    "args": {
      "External id": 2481030,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568079.300, "dur": 3.970,
    "args": {
      "External id": 2481031,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568095.528, "dur": 8.082,
    "args": {
      "External id": 2481032,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568098.433, "dur": 4.893,
    "args": {
      "External id": 2481033,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568254.475, "dur": 258.562,
    "args": {
      "External id": 2481034,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568259.842, "dur": 2.598,
    "args": {
      "External id": 2481035,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568264.838, "dur": 247.746,
    "args": {
      "External id": 2481036,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568269.759, "dur": 0.684,
    "args": {
      "External id": 2481037,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568272.888, "dur": 31.651,
    "args": {
      "External id": 2481038,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568307.975, "dur": 5.570,
    "args": {
      "External id": 2481039,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568312.290, "dur": 0.906,
    "args": {
      "External id": 2481040,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568315.281, "dur": 29.237,
    "args": {
      "External id": 2481041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568316.668, "dur": 1.682,
    "args": {
      "External id": 2481042,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568320.463, "dur": 23.749,
    "args": {
      "External id": 2481043,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568325.180, "dur": 3.674,
    "args": {
      "External id": 2481044,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568347.112, "dur": 26.905,
    "args": {
      "External id": 2481045,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568376.831, "dur": 24.579,
    "args": {
      "External id": 2481046,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568408.032, "dur": 19.406,
    "args": {
      "External id": 2481047,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568430.216, "dur": 17.746,
    "args": {
      "External id": 2481048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568450.224, "dur": 25.313,
    "args": {
      "External id": 2481049,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568452.680, "dur": 1.912,
    "args": {
      "External id": 2481050,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568456.505, "dur": 1.337,
    "args": {
      "External id": 2481051,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568478.579, "dur": 15.692,
    "args": {
      "External id": 2481052,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568498.922, "dur": 12.378,
    "args": {
      "External id": 2481053,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568520.122, "dur": 2.001,
    "args": {
      "External id": 2481054,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568541.886, "dur": 4.918,
    "args": {
      "External id": 2481055,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568544.604, "dur": 0.725,
    "args": {
      "External id": 2481056,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568638.202, "dur": 82.392,
    "args": {
      "External id": 2481057,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568725.670, "dur": 5.153,
    "args": {
      "External id": 2481058,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568728.339, "dur": 1.117,
    "args": {
      "External id": 2481059,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568732.869, "dur": 33.264,
    "args": {
      "External id": 2481060,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568774.663, "dur": 7.046,
    "args": {
      "External id": 2481061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568776.976, "dur": 4.072,
    "args": {
      "External id": 2481062,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568778.920, "dur": 1.838,
    "args": {
      "External id": 2481063,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568785.771, "dur": 51.092,
    "args": {
      "External id": 2481064,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568787.290, "dur": 48.938,
    "args": {
      "External id": 2481065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568842.731, "dur": 19.185,
    "args": {
      "External id": 2481066,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568867.603, "dur": 6.332,
    "args": {
      "External id": 2481067,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568871.991, "dur": 0.897,
    "args": {
      "External id": 2481068,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568878.771, "dur": 53.192,
    "args": {
      "External id": 2481069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568879.694, "dur": 3.334,
    "args": {
      "External id": 2481070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568880.493, "dur": 1.993,
    "args": {
      "External id": 2481071,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568881.641, "dur": 0.707,
    "args": {
      "External id": 2481072,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568883.715, "dur": 47.731,
    "args": {
      "External id": 2481073,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568886.645, "dur": 44.165,
    "args": {
      "External id": 2481074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568936.015, "dur": 5.744,
    "args": {
      "External id": 2481075,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568938.270, "dur": 2.486,
    "args": {
      "External id": 2481076,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568947.995, "dur": 1.795,
    "args": {
      "External id": 2481077,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568957.913, "dur": 8.717,
    "args": {
      "External id": 2481078,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656568961.952, "dur": 4.374,
    "args": {
      "External id": 2481079,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569105.689, "dur": 206.247,
    "args": {
      "External id": 2481080,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569108.265, "dur": 3.089,
    "args": {
      "External id": 2481081,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569113.739, "dur": 197.649,
    "args": {
      "External id": 2481082,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569115.067, "dur": 0.507,
    "args": {
      "External id": 2481083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569116.789, "dur": 28.253,
    "args": {
      "External id": 2481084,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569146.820, "dur": 3.974,
    "args": {
      "External id": 2481085,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569149.206, "dur": 1.257,
    "args": {
      "External id": 2481086,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569151.696, "dur": 27.922,
    "args": {
      "External id": 2481087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569155.598, "dur": 1.614,
    "args": {
      "External id": 2481088,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569158.210, "dur": 21.076,
    "args": {
      "External id": 2481089,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569163.027, "dur": 3.373,
    "args": {
      "External id": 2481090,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569181.162, "dur": 25.001,
    "args": {
      "External id": 2481091,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569208.093, "dur": 14.057,
    "args": {
      "External id": 2481092,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569225.108, "dur": 16.790,
    "args": {
      "External id": 2481093,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569243.435, "dur": 13.481,
    "args": {
      "External id": 2481094,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569258.733, "dur": 23.542,
    "args": {
      "External id": 2481095,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569260.929, "dur": 1.545,
    "args": {
      "External id": 2481096,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569266.377, "dur": 0.838,
    "args": {
      "External id": 2481097,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569283.959, "dur": 13.413,
    "args": {
      "External id": 2481098,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569298.484, "dur": 12.039,
    "args": {
      "External id": 2481099,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569319.879, "dur": 2.169,
    "args": {
      "External id": 2481100,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569331.590, "dur": 3.532,
    "args": {
      "External id": 2481101,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569333.652, "dur": 0.682,
    "args": {
      "External id": 2481102,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569402.546, "dur": 59.910,
    "args": {
      "External id": 2481103,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569467.269, "dur": 6.911,
    "args": {
      "External id": 2481104,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569471.670, "dur": 1.533,
    "args": {
      "External id": 2481105,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569475.684, "dur": 26.383,
    "args": {
      "External id": 2481106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569506.859, "dur": 5.599,
    "args": {
      "External id": 2481107,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569508.568, "dur": 3.298,
    "args": {
      "External id": 2481108,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569510.750, "dur": 0.912,
    "args": {
      "External id": 2481109,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569515.040, "dur": 67.139,
    "args": {
      "External id": 2481110,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569518.364, "dur": 62.927,
    "args": {
      "External id": 2481111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569587.798, "dur": 20.396,
    "args": {
      "External id": 2481112,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569614.037, "dur": 4.525,
    "args": {
      "External id": 2481113,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569616.615, "dur": 1.002,
    "args": {
      "External id": 2481114,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569622.822, "dur": 57.185,
    "args": {
      "External id": 2481115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569624.006, "dur": 8.840,
    "args": {
      "External id": 2481116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569625.090, "dur": 6.927,
    "args": {
      "External id": 2481117,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569631.145, "dur": 0.737,
    "args": {
      "External id": 2481118,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569633.746, "dur": 45.732,
    "args": {
      "External id": 2481119,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569634.333, "dur": 44.363,
    "args": {
      "External id": 2481120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569684.099, "dur": 4.127,
    "args": {
      "External id": 2481121,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569686.170, "dur": 1.047,
    "args": {
      "External id": 2481122,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569694.637, "dur": 1.713,
    "args": {
      "External id": 2481123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569704.317, "dur": 8.184,
    "args": {
      "External id": 2481124,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569708.242, "dur": 4.003,
    "args": {
      "External id": 2481125,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569806.627, "dur": 328.104,
    "args": {
      "External id": 2481126,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569808.922, "dur": 2.360,
    "args": {
      "External id": 2481127,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569812.692, "dur": 321.441,
    "args": {
      "External id": 2481128,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569816.174, "dur": 0.360,
    "args": {
      "External id": 2481129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569817.600, "dur": 21.859,
    "args": {
      "External id": 2481130,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569841.205, "dur": 3.049,
    "args": {
      "External id": 2481131,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569843.183, "dur": 0.829,
    "args": {
      "External id": 2481132,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569845.024, "dur": 28.852,
    "args": {
      "External id": 2481133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569850.616, "dur": 1.529,
    "args": {
      "External id": 2481134,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569853.225, "dur": 20.332,
    "args": {
      "External id": 2481135,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569855.952, "dur": 2.626,
    "args": {
      "External id": 2481136,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569875.301, "dur": 32.608,
    "args": {
      "External id": 2481137,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569909.642, "dur": 39.496,
    "args": {
      "External id": 2481138,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656569952.277, "dur": 63.733,
    "args": {
      "External id": 2481139,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570018.466, "dur": 36.597,
    "args": {
      "External id": 2481140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570056.844, "dur": 28.703,
    "args": {
      "External id": 2481141,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570058.904, "dur": 2.179,
    "args": {
      "External id": 2481142,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570065.088, "dur": 0.973,
    "args": {
      "External id": 2481143,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570087.327, "dur": 25.954,
    "args": {
      "External id": 2481144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570114.503, "dur": 18.626,
    "args": {
      "External id": 2481145,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570142.504, "dur": 2.176,
    "args": {
      "External id": 2481146,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570153.496, "dur": 3.780,
    "args": {
      "External id": 2481147,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570155.731, "dur": 0.523,
    "args": {
      "External id": 2481148,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570231.075, "dur": 58.211,
    "args": {
      "External id": 2481149,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570296.121, "dur": 4.710,
    "args": {
      "External id": 2481150,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570298.592, "dur": 1.037,
    "args": {
      "External id": 2481151,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570302.173, "dur": 25.280,
    "args": {
      "External id": 2481152,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570374.568, "dur": 8.442,
    "args": {
      "External id": 2481153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570376.424, "dur": 5.901,
    "args": {
      "External id": 2481154,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570378.852, "dur": 3.206,
    "args": {
      "External id": 2481155,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570387.821, "dur": 44.046,
    "args": {
      "External id": 2481156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570388.825, "dur": 42.286,
    "args": {
      "External id": 2481157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570436.172, "dur": 16.691,
    "args": {
      "External id": 2481158,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570458.466, "dur": 3.426,
    "args": {
      "External id": 2481159,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570460.447, "dur": 0.623,
    "args": {
      "External id": 2481160,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570465.931, "dur": 50.617,
    "args": {
      "External id": 2481161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570466.706, "dur": 5.841,
    "args": {
      "External id": 2481162,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570469.449, "dur": 2.555,
    "args": {
      "External id": 2481163,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570470.841, "dur": 0.843,
    "args": {
      "External id": 2481164,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570473.218, "dur": 42.895,
    "args": {
      "External id": 2481165,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570473.699, "dur": 41.812,
    "args": {
      "External id": 2481166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570520.267, "dur": 3.916,
    "args": {
      "External id": 2481167,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570522.267, "dur": 0.785,
    "args": {
      "External id": 2481168,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570547.824, "dur": 2.221,
    "args": {
      "External id": 2481169,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570558.533, "dur": 7.855,
    "args": {
      "External id": 2481170,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570560.503, "dur": 5.653,
    "args": {
      "External id": 2481171,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570660.025, "dur": 198.736,
    "args": {
      "External id": 2481172,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570662.086, "dur": 2.328,
    "args": {
      "External id": 2481173,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570668.393, "dur": 189.923,
    "args": {
      "External id": 2481174,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570671.616, "dur": 0.385,
    "args": {
      "External id": 2481175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570673.294, "dur": 23.393,
    "args": {
      "External id": 2481176,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570698.278, "dur": 4.695,
    "args": {
      "External id": 2481177,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570700.030, "dur": 2.660,
    "args": {
      "External id": 2481178,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570703.764, "dur": 23.150,
    "args": {
      "External id": 2481179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570705.303, "dur": 1.794,
    "args": {
      "External id": 2481180,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570708.103, "dur": 18.405,
    "args": {
      "External id": 2481181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570710.676, "dur": 2.730,
    "args": {
      "External id": 2481182,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570728.475, "dur": 23.808,
    "args": {
      "External id": 2481183,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570753.896, "dur": 14.065,
    "args": {
      "External id": 2481184,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570772.830, "dur": 16.171,
    "args": {
      "External id": 2481185,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570790.222, "dur": 14.222,
    "args": {
      "External id": 2481186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570805.979, "dur": 23.436,
    "args": {
      "External id": 2481187,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570807.605, "dur": 1.657,
    "args": {
      "External id": 2481188,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570811.267, "dur": 2.887,
    "args": {
      "External id": 2481189,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570830.828, "dur": 13.106,
    "args": {
      "External id": 2481190,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570844.963, "dur": 12.510,
    "args": {
      "External id": 2481191,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570867.433, "dur": 1.612,
    "args": {
      "External id": 2481192,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570877.874, "dur": 3.387,
    "args": {
      "External id": 2481193,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570880.051, "dur": 0.416,
    "args": {
      "External id": 2481194,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656570945.634, "dur": 96.596,
    "args": {
      "External id": 2481195,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571049.206, "dur": 6.455,
    "args": {
      "External id": 2481196,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571052.763, "dur": 1.190,
    "args": {
      "External id": 2481197,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571057.068, "dur": 28.438,
    "args": {
      "External id": 2481198,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571090.978, "dur": 8.125,
    "args": {
      "External id": 2481199,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571094.672, "dur": 3.721,
    "args": {
      "External id": 2481200,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571096.571, "dur": 1.593,
    "args": {
      "External id": 2481201,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571102.025, "dur": 46.353,
    "args": {
      "External id": 2481202,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571102.885, "dur": 44.979,
    "args": {
      "External id": 2481203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571152.858, "dur": 16.629,
    "args": {
      "External id": 2481204,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571175.085, "dur": 6.371,
    "args": {
      "External id": 2481205,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571179.737, "dur": 0.775,
    "args": {
      "External id": 2481206,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571185.512, "dur": 48.952,
    "args": {
      "External id": 2481207,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571186.396, "dur": 3.837,
    "args": {
      "External id": 2481208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571187.273, "dur": 2.447,
    "args": {
      "External id": 2481209,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571188.553, "dur": 1.054,
    "args": {
      "External id": 2481210,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571190.846, "dur": 43.218,
    "args": {
      "External id": 2481211,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571191.845, "dur": 41.705,
    "args": {
      "External id": 2481212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571238.925, "dur": 4.342,
    "args": {
      "External id": 2481213,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571240.781, "dur": 1.149,
    "args": {
      "External id": 2481214,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571251.471, "dur": 1.463,
    "args": {
      "External id": 2481215,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571261.102, "dur": 6.313,
    "args": {
      "External id": 2481216,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571263.267, "dur": 3.884,
    "args": {
      "External id": 2481217,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571358.833, "dur": 224.345,
    "args": {
      "External id": 2481218,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571363.304, "dur": 2.153,
    "args": {
      "External id": 2481219,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571369.056, "dur": 213.617,
    "args": {
      "External id": 2481220,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571370.164, "dur": 0.551,
    "args": {
      "External id": 2481221,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571371.695, "dur": 22.586,
    "args": {
      "External id": 2481222,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571395.929, "dur": 4.840,
    "args": {
      "External id": 2481223,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571399.753, "dur": 0.792,
    "args": {
      "External id": 2481224,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571401.652, "dur": 22.178,
    "args": {
      "External id": 2481225,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571402.753, "dur": 1.579,
    "args": {
      "External id": 2481226,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571405.346, "dur": 18.135,
    "args": {
      "External id": 2481227,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571408.179, "dur": 2.476,
    "args": {
      "External id": 2481228,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571427.192, "dur": 21.460,
    "args": {
      "External id": 2481229,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571450.248, "dur": 17.649,
    "args": {
      "External id": 2481230,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571470.754, "dur": 14.511,
    "args": {
      "External id": 2481231,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571486.551, "dur": 14.506,
    "args": {
      "External id": 2481232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571502.527, "dur": 20.216,
    "args": {
      "External id": 2481233,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571504.632, "dur": 1.397,
    "args": {
      "External id": 2481234,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571507.715, "dur": 1.390,
    "args": {
      "External id": 2481235,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571524.230, "dur": 32.890,
    "args": {
      "External id": 2481236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571561.201, "dur": 20.112,
    "args": {
      "External id": 2481237,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571591.232, "dur": 2.647,
    "args": {
      "External id": 2481238,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571602.770, "dur": 4.016,
    "args": {
      "External id": 2481239,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571605.168, "dur": 0.843,
    "args": {
      "External id": 2481240,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571678.190, "dur": 56.181,
    "args": {
      "External id": 2481241,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571739.435, "dur": 4.805,
    "args": {
      "External id": 2481242,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571741.882, "dur": 1.115,
    "args": {
      "External id": 2481243,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571745.969, "dur": 23.441,
    "args": {
      "External id": 2481244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571776.012, "dur": 5.312,
    "args": {
      "External id": 2481245,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571777.355, "dur": 3.184,
    "args": {
      "External id": 2481246,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571779.088, "dur": 1.246,
    "args": {
      "External id": 2481247,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571783.718, "dur": 41.530,
    "args": {
      "External id": 2481248,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571784.915, "dur": 39.704,
    "args": {
      "External id": 2481249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571828.754, "dur": 16.450,
    "args": {
      "External id": 2481250,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571850.560, "dur": 5.872,
    "args": {
      "External id": 2481251,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571854.786, "dur": 0.879,
    "args": {
      "External id": 2481252,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571859.892, "dur": 50.888,
    "args": {
      "External id": 2481253,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571861.127, "dur": 4.016,
    "args": {
      "External id": 2481254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571862.195, "dur": 2.470,
    "args": {
      "External id": 2481255,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571863.439, "dur": 1.111,
    "args": {
      "External id": 2481256,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571865.619, "dur": 44.734,
    "args": {
      "External id": 2481257,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571868.214, "dur": 41.522,
    "args": {
      "External id": 2481258,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571914.369, "dur": 4.108,
    "args": {
      "External id": 2481259,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571916.612, "dur": 0.851,
    "args": {
      "External id": 2481260,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571933.311, "dur": 1.345,
    "args": {
      "External id": 2481261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571946.578, "dur": 10.300,
    "args": {
      "External id": 2481262,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656571950.587, "dur": 5.988,
    "args": {
      "External id": 2481263,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572082.403, "dur": 210.190,
    "args": {
      "External id": 2481264,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572084.882, "dur": 3.226,
    "args": {
      "External id": 2481265,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572090.042, "dur": 202.093,
    "args": {
      "External id": 2481266,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572101.128, "dur": 0.460,
    "args": {
      "External id": 2481267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572104.656, "dur": 24.976,
    "args": {
      "External id": 2481268,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572131.318, "dur": 5.288,
    "args": {
      "External id": 2481269,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572133.213, "dur": 3.008,
    "args": {
      "External id": 2481270,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572137.620, "dur": 24.697,
    "args": {
      "External id": 2481271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572140.779, "dur": 1.664,
    "args": {
      "External id": 2481272,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572143.484, "dur": 18.435,
    "args": {
      "External id": 2481273,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572146.152, "dur": 3.032,
    "args": {
      "External id": 2481274,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572163.728, "dur": 23.295,
    "args": {
      "External id": 2481275,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572188.396, "dur": 14.460,
    "args": {
      "External id": 2481276,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572206.007, "dur": 16.575,
    "args": {
      "External id": 2481277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572223.962, "dur": 13.786,
    "args": {
      "External id": 2481278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572239.373, "dur": 23.538,
    "args": {
      "External id": 2481279,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572241.282, "dur": 1.994,
    "args": {
      "External id": 2481280,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572247.573, "dur": 0.934,
    "args": {
      "External id": 2481281,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572264.410, "dur": 13.724,
    "args": {
      "External id": 2481282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572279.077, "dur": 11.968,
    "args": {
      "External id": 2481283,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572299.728, "dur": 1.945,
    "args": {
      "External id": 2481284,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572310.856, "dur": 3.714,
    "args": {
      "External id": 2481285,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572313.351, "dur": 0.407,
    "args": {
      "External id": 2481286,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572374.698, "dur": 57.624,
    "args": {
      "External id": 2481287,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572438.711, "dur": 6.610,
    "args": {
      "External id": 2481288,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572441.276, "dur": 2.804,
    "args": {
      "External id": 2481289,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572446.724, "dur": 25.244,
    "args": {
      "External id": 2481290,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572476.512, "dur": 5.485,
    "args": {
      "External id": 2481291,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572478.150, "dur": 3.266,
    "args": {
      "External id": 2481292,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572480.192, "dur": 1.024,
    "args": {
      "External id": 2481293,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572486.915, "dur": 55.685,
    "args": {
      "External id": 2481294,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572487.812, "dur": 38.526,
    "args": {
      "External id": 2481295,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572548.410, "dur": 18.021,
    "args": {
      "External id": 2481296,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572572.474, "dur": 5.200,
    "args": {
      "External id": 2481297,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572575.350, "dur": 1.415,
    "args": {
      "External id": 2481298,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572581.677, "dur": 54.398,
    "args": {
      "External id": 2481299,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572582.560, "dur": 6.204,
    "args": {
      "External id": 2481300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572585.601, "dur": 2.688,
    "args": {
      "External id": 2481301,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572587.396, "dur": 0.768,
    "args": {
      "External id": 2481302,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572589.564, "dur": 46.136,
    "args": {
      "External id": 2481303,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572590.138, "dur": 44.839,
    "args": {
      "External id": 2481304,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572640.181, "dur": 3.997,
    "args": {
      "External id": 2481305,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572642.029, "dur": 0.796,
    "args": {
      "External id": 2481306,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572650.458, "dur": 1.433,
    "args": {
      "External id": 2481307,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572662.158, "dur": 6.375,
    "args": {
      "External id": 2481308,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572664.449, "dur": 3.858,
    "args": {
      "External id": 2481309,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572751.672, "dur": 189.586,
    "args": {
      "External id": 2481310,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572753.924, "dur": 4.349,
    "args": {
      "External id": 2481311,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572759.502, "dur": 181.250,
    "args": {
      "External id": 2481312,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572760.855, "dur": 0.318,
    "args": {
      "External id": 2481313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572762.457, "dur": 21.653,
    "args": {
      "External id": 2481314,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572788.047, "dur": 6.087,
    "args": {
      "External id": 2481315,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572792.821, "dur": 1.007,
    "args": {
      "External id": 2481316,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572795.059, "dur": 21.605,
    "args": {
      "External id": 2481317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572796.081, "dur": 1.838,
    "args": {
      "External id": 2481318,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572798.853, "dur": 17.525,
    "args": {
      "External id": 2481319,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572801.307, "dur": 2.341,
    "args": {
      "External id": 2481320,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572817.912, "dur": 20.829,
    "args": {
      "External id": 2481321,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572840.178, "dur": 14.040,
    "args": {
      "External id": 2481322,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572856.831, "dur": 14.591,
    "args": {
      "External id": 2481323,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572872.630, "dur": 14.090,
    "args": {
      "External id": 2481324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572890.201, "dur": 22.494,
    "args": {
      "External id": 2481325,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572892.448, "dur": 1.381,
    "args": {
      "External id": 2481326,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572895.472, "dur": 2.948,
    "args": {
      "External id": 2481327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572914.049, "dur": 13.568,
    "args": {
      "External id": 2481328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572928.656, "dur": 11.180,
    "args": {
      "External id": 2481329,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572947.499, "dur": 1.420,
    "args": {
      "External id": 2481330,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572956.849, "dur": 3.389,
    "args": {
      "External id": 2481331,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656572959.109, "dur": 0.341,
    "args": {
      "External id": 2481332,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573066.309, "dur": 58.081,
    "args": {
      "External id": 2481333,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573130.144, "dur": 5.834,
    "args": {
      "External id": 2481334,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573133.447, "dur": 1.088,
    "args": {
      "External id": 2481335,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573137.453, "dur": 25.837,
    "args": {
      "External id": 2481336,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573168.682, "dur": 7.701,
    "args": {
      "External id": 2481337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573170.072, "dur": 5.649,
    "args": {
      "External id": 2481338,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573173.941, "dur": 1.541,
    "args": {
      "External id": 2481339,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573179.278, "dur": 41.656,
    "args": {
      "External id": 2481340,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573180.466, "dur": 39.733,
    "args": {
      "External id": 2481341,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573225.130, "dur": 16.549,
    "args": {
      "External id": 2481342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573247.056, "dur": 3.870,
    "args": {
      "External id": 2481343,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573249.363, "dur": 0.724,
    "args": {
      "External id": 2481344,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573254.580, "dur": 50.867,
    "args": {
      "External id": 2481345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573257.380, "dur": 3.868,
    "args": {
      "External id": 2481346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573258.272, "dur": 2.391,
    "args": {
      "External id": 2481347,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573259.651, "dur": 0.872,
    "args": {
      "External id": 2481348,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573262.028, "dur": 43.022,
    "args": {
      "External id": 2481349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573263.094, "dur": 41.394,
    "args": {
      "External id": 2481350,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573309.274, "dur": 3.483,
    "args": {
      "External id": 2481351,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573311.119, "dur": 0.748,
    "args": {
      "External id": 2481352,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573320.350, "dur": 1.570,
    "args": {
      "External id": 2481353,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573329.193, "dur": 6.195,
    "args": {
      "External id": 2481354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573331.062, "dur": 4.092,
    "args": {
      "External id": 2481355,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573418.617, "dur": 212.348,
    "args": {
      "External id": 2481356,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573422.837, "dur": 2.180,
    "args": {
      "External id": 2481357,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573428.483, "dur": 201.617,
    "args": {
      "External id": 2481358,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573429.607, "dur": 0.350,
    "args": {
      "External id": 2481359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573430.910, "dur": 22.857,
    "args": {
      "External id": 2481360,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573455.424, "dur": 4.832,
    "args": {
      "External id": 2481361,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573459.124, "dur": 0.925,
    "args": {
      "External id": 2481362,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573461.060, "dur": 21.448,
    "args": {
      "External id": 2481363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573462.085, "dur": 1.420,
    "args": {
      "External id": 2481364,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573464.459, "dur": 17.748,
    "args": {
      "External id": 2481365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573466.659, "dur": 2.370,
    "args": {
      "External id": 2481366,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573485.817, "dur": 21.165,
    "args": {
      "External id": 2481367,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573508.236, "dur": 13.559,
    "args": {
      "External id": 2481368,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573524.412, "dur": 32.898,
    "args": {
      "External id": 2481369,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573559.492, "dur": 13.536,
    "args": {
      "External id": 2481370,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573574.788, "dur": 24.324,
    "args": {
      "External id": 2481371,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573578.532, "dur": 2.149,
    "args": {
      "External id": 2481372,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573583.029, "dur": 0.859,
    "args": {
      "External id": 2481373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573600.398, "dur": 13.143,
    "args": {
      "External id": 2481374,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573616.963, "dur": 12.083,
    "args": {
      "External id": 2481375,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573638.197, "dur": 2.221,
    "args": {
      "External id": 2481376,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573648.944, "dur": 4.002,
    "args": {
      "External id": 2481377,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573651.512, "dur": 0.663,
    "args": {
      "External id": 2481378,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573715.942, "dur": 56.539,
    "args": {
      "External id": 2481379,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573777.235, "dur": 4.353,
    "args": {
      "External id": 2481380,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573779.661, "dur": 1.027,
    "args": {
      "External id": 2481381,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573783.072, "dur": 24.549,
    "args": {
      "External id": 2481382,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573813.715, "dur": 5.568,
    "args": {
      "External id": 2481383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573814.996, "dur": 3.663,
    "args": {
      "External id": 2481384,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573817.306, "dur": 1.180,
    "args": {
      "External id": 2481385,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573821.815, "dur": 40.260,
    "args": {
      "External id": 2481386,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573822.926, "dur": 38.557,
    "args": {
      "External id": 2481387,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573865.499, "dur": 15.130,
    "args": {
      "External id": 2481388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573884.738, "dur": 27.442,
    "args": {
      "External id": 2481389,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573889.481, "dur": 22.332,
    "args": {
      "External id": 2481390,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573894.744, "dur": 0.714,
    "args": {
      "External id": 2481391,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573917.545, "dur": 33.030,
    "args": {
      "External id": 2481392,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573919.503, "dur": 30.856,
    "args": {
      "External id": 2481393,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573924.350, "dur": 4.535,
    "args": {
      "External id": 2481394,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573929.887, "dur": 20.030,
    "args": {
      "External id": 2481395,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573964.002, "dur": 5.838,
    "args": {
      "External id": 2481396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573966.201, "dur": 3.349,
    "args": {
      "External id": 2481397,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573970.836, "dur": 3.653,
    "args": {
      "External id": 2481398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656573973.888, "dur": 0.504,
    "args": {
      "External id": 2481399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574051.705, "dur": 28.612,
    "args": {
      "External id": 2481400,"Sequence number": 24740625, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9103
    }
  },
  {
    "ph": "s", "id": 23, "pid": 1336760, "tid": 1336760, "ts": 1514656574051.705,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574086.613, "dur": 7.816,
    "args": {
      "External id": 2481401,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574091.880, "dur": 1.179,
    "args": {
      "External id": 2481402,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574096.832, "dur": 6.823,
    "args": {
      "External id": 2481403,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "1"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574101.518, "dur": 0.862,
    "args": {
      "External id": 2481404,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574107.107, "dur": 3.562,
    "args": {
      "External id": 2481405,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574109.301, "dur": 0.779,
    "args": {
      "External id": 2481406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574114.852, "dur": 7.141,
    "args": {
      "External id": 2481407,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9110
    }
  },
  {
    "ph": "s", "id": 22, "pid": 1336760, "tid": 1336760, "ts": 1514656574114.852,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574119.583, "dur": 1.143,
    "args": {
      "External id": 2481408,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574123.082, "dur": 6.652,
    "args": {
      "External id": 2481409,"Sequence number": 24740627, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9112
    }
  },
  {
    "ph": "s", "id": 21, "pid": 1336760, "tid": 1336760, "ts": 1514656574123.082,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574128.215, "dur": 0.815,
    "args": {
      "External id": 2481410,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574130.767, "dur": 5.728,
    "args": {
      "External id": 2481411,"Sequence number": 24740628, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9114
    }
  },
  {
    "ph": "s", "id": 20, "pid": 1336760, "tid": 1336760, "ts": 1514656574130.767,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574134.717, "dur": 0.903,
    "args": {
      "External id": 2481412,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574137.666, "dur": 4.773,
    "args": {
      "External id": 2481413,"Sequence number": 24740629, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9116
    }
  },
  {
    "ph": "s", "id": 19, "pid": 1336760, "tid": 1336760, "ts": 1514656574137.666,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574140.480, "dur": 1.249,
    "args": {
      "External id": 2481414,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574146.263, "dur": 36.221,
    "args": {
      "External id": 2481415,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574147.876, "dur": 34.391,
    "args": {
      "External id": 2481416,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574150.487, "dur": 11.488,
    "args": {
      "External id": 2481417,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574155.250, "dur": 6.165,
    "args": {
      "External id": 2481418,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574163.008, "dur": 18.788,
    "args": {
      "External id": 2481419,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574207.701, "dur": 4.697,
    "args": {
      "External id": 2481420,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9123
    }
  },
  {
    "ph": "s", "id": 18, "pid": 1336760, "tid": 1336760, "ts": 1514656574207.701,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574214.787, "dur": 1.507,
    "args": {
      "External id": 2481421,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574248.249, "dur": 42422.223,
    "args": {
      "External id": 2481422,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9125
    }
  },
  {
    "ph": "s", "id": 17, "pid": 1336760, "tid": 1336760, "ts": 1514656574248.249,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574262.039, "dur": 28.595,
    "args": {
      "External id": 2481423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574262.702, "dur": 27.669,
    "args": {
      "External id": 2481424,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574264.491, "dur": 7.625,
    "args": {
      "External id": 2481425,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574267.993, "dur": 3.772,
    "args": {
      "External id": 2481426,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574272.730, "dur": 17.323,
    "args": {
      "External id": 2481427,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574305.337, "dur": 25.946,
    "args": {
      "External id": 2481428,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574306.482, "dur": 6.147,
    "args": {
      "External id": 2481429,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574308.740, "dur": 3.595,
    "args": {
      "External id": 2481430,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574314.136, "dur": 16.935,
    "args": {
      "External id": 2481431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574315.934, "dur": 14.738,
    "args": {
      "External id": 2481432,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574334.540, "dur": 24.325,
    "args": {
      "External id": 2481433,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574335.434, "dur": 4.414,
    "args": {
      "External id": 2481434,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574336.632, "dur": 2.975,
    "args": {
      "External id": 2481435,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574340.591, "dur": 18.065,
    "args": {
      "External id": 2481436,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574343.035, "dur": 15.270,
    "args": {
      "External id": 2481437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574364.883, "dur": 19.047,
    "args": {
      "External id": 2481438,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574366.492, "dur": 3.539,
    "args": {
      "External id": 2481439,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574370.744, "dur": 12.911,
    "args": {
      "External id": 2481440,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574371.435, "dur": 11.857,
    "args": {
      "External id": 2481441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574388.600, "dur": 23.495,
    "args": {
      "External id": 2481442,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574414.798, "dur": 72.561,
    "args": {
      "External id": 2481443,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574435.461, "dur": 51.534,
    "args": {
      "External id": 2481444,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574440.300, "dur": 2.950,
    "args": {
      "External id": 2481445,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574444.508, "dur": 26.072,
    "args": {
      "External id": 2481446,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574448.156, "dur": 22.015,
    "args": {
      "External id": 2481447,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574450.791, "dur": 2.824,
    "args": {
      "External id": 2481448,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574454.611, "dur": 15.215,
    "args": {
      "External id": 2481449,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574490.891, "dur": 36354.835,
    "args": {
      "External id": 2481450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514656574492.278, "dur": 36352.704,
    "args": {
      "External id": 2481451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610855.213, "dur": 6.215,
    "args": {
      "External id": 2481452,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610858.615, "dur": 1.174,
    "args": {
      "External id": 2481453,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610866.305, "dur": 104.089,
    "args": {
      "External id": 2481454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610870.331, "dur": 6.162,
    "args": {
      "External id": 2481455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610872.587, "dur": 3.024,
    "args": {
      "External id": 2481456,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610874.450, "dur": 0.916,
    "args": {
      "External id": 2481457,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610877.912, "dur": 91.800,
    "args": {
      "External id": 2481458,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610879.567, "dur": 89.379,
    "args": {
      "External id": 2481459,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610973.877, "dur": 4.362,
    "args": {
      "External id": 2481460,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656610976.115, "dur": 1.026,
    "args": {
      "External id": 2481461,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611011.213, "dur": 2.848,
    "args": {
      "External id": 2481462,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611023.204, "dur": 7.141,
    "args": {
      "External id": 2481463,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611025.099, "dur": 4.934,
    "args": {
      "External id": 2481464,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611154.859, "dur": 217.030,
    "args": {
      "External id": 2481465,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611159.705, "dur": 2.471,
    "args": {
      "External id": 2481466,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611164.402, "dur": 206.804,
    "args": {
      "External id": 2481467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611168.312, "dur": 0.456,
    "args": {
      "External id": 2481468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611170.468, "dur": 26.380,
    "args": {
      "External id": 2481469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611198.956, "dur": 5.570,
    "args": {
      "External id": 2481470,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611203.484, "dur": 0.682,
    "args": {
      "External id": 2481471,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611205.671, "dur": 25.196,
    "args": {
      "External id": 2481472,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611206.881, "dur": 1.476,
    "args": {
      "External id": 2481473,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611209.569, "dur": 21.020,
    "args": {
      "External id": 2481474,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611213.019, "dur": 3.190,
    "args": {
      "External id": 2481475,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611232.816, "dur": 23.353,
    "args": {
      "External id": 2481476,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611258.217, "dur": 18.667,
    "args": {
      "External id": 2481477,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611282.480, "dur": 16.970,
    "args": {
      "External id": 2481478,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611301.035, "dur": 14.111,
    "args": {
      "External id": 2481479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611316.843, "dur": 22.503,
    "args": {
      "External id": 2481480,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611319.535, "dur": 1.604,
    "args": {
      "External id": 2481481,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611323.543, "dur": 0.671,
    "args": {
      "External id": 2481482,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611340.950, "dur": 13.386,
    "args": {
      "External id": 2481483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611358.189, "dur": 11.906,
    "args": {
      "External id": 2481484,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611379.030, "dur": 2.267,
    "args": {
      "External id": 2481485,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611388.067, "dur": 3.943,
    "args": {
      "External id": 2481486,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611390.588, "dur": 0.588,
    "args": {
      "External id": 2481487,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611458.883, "dur": 81.939,
    "args": {
      "External id": 2481488,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611548.243, "dur": 7.477,
    "args": {
      "External id": 2481489,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611551.346, "dur": 1.485,
    "args": {
      "External id": 2481490,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611557.366, "dur": 33.839,
    "args": {
      "External id": 2481491,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611599.306, "dur": 6.432,
    "args": {
      "External id": 2481492,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611601.284, "dur": 3.672,
    "args": {
      "External id": 2481493,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611603.387, "dur": 1.348,
    "args": {
      "External id": 2481494,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611609.078, "dur": 49.241,
    "args": {
      "External id": 2481495,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611610.400, "dur": 47.041,
    "args": {
      "External id": 2481496,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611662.373, "dur": 16.885,
    "args": {
      "External id": 2481497,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611685.231, "dur": 6.248,
    "args": {
      "External id": 2481498,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611689.619, "dur": 0.974,
    "args": {
      "External id": 2481499,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611696.339, "dur": 58.206,
    "args": {
      "External id": 2481500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611697.610, "dur": 3.884,
    "args": {
      "External id": 2481501,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611698.545, "dur": 2.420,
    "args": {
      "External id": 2481502,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611699.937, "dur": 0.860,
    "args": {
      "External id": 2481503,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611702.377, "dur": 51.768,
    "args": {
      "External id": 2481504,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611710.518, "dur": 43.043,
    "args": {
      "External id": 2481505,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611758.903, "dur": 4.196,
    "args": {
      "External id": 2481506,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611761.116, "dur": 0.899,
    "args": {
      "External id": 2481507,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611769.147, "dur": 1.774,
    "args": {
      "External id": 2481508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611779.136, "dur": 11.018,
    "args": {
      "External id": 2481509,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611783.744, "dur": 6.165,
    "args": {
      "External id": 2481510,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611881.603, "dur": 238.814,
    "args": {
      "External id": 2481511,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611883.931, "dur": 2.181,
    "args": {
      "External id": 2481512,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611887.692, "dur": 232.137,
    "args": {
      "External id": 2481513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611889.234, "dur": 0.322,
    "args": {
      "External id": 2481514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611890.760, "dur": 25.258,
    "args": {
      "External id": 2481515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611919.918, "dur": 6.714,
    "args": {
      "External id": 2481516,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611925.598, "dur": 0.644,
    "args": {
      "External id": 2481517,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611929.585, "dur": 24.033,
    "args": {
      "External id": 2481518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611931.389, "dur": 3.401,
    "args": {
      "External id": 2481519,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611936.142, "dur": 17.216,
    "args": {
      "External id": 2481520,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611938.675, "dur": 2.453,
    "args": {
      "External id": 2481521,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611955.033, "dur": 22.403,
    "args": {
      "External id": 2481522,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656611978.971, "dur": 49.942,
    "args": {
      "External id": 2481523,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612033.229, "dur": 17.063,
    "args": {
      "External id": 2481524,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612051.595, "dur": 13.419,
    "args": {
      "External id": 2481525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612066.858, "dur": 24.056,
    "args": {
      "External id": 2481526,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612071.226, "dur": 1.608,
    "args": {
      "External id": 2481527,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612074.824, "dur": 1.035,
    "args": {
      "External id": 2481528,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612092.396, "dur": 13.720,
    "args": {
      "External id": 2481529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612107.142, "dur": 11.646,
    "args": {
      "External id": 2481530,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612128.501, "dur": 2.137,
    "args": {
      "External id": 2481531,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612139.694, "dur": 3.624,
    "args": {
      "External id": 2481532,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612141.931, "dur": 0.606,
    "args": {
      "External id": 2481533,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612210.705, "dur": 57.729,
    "args": {
      "External id": 2481534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612275.576, "dur": 4.660,
    "args": {
      "External id": 2481535,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612278.260, "dur": 1.072,
    "args": {
      "External id": 2481536,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612281.787, "dur": 25.562,
    "args": {
      "External id": 2481537,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612311.915, "dur": 7.540,
    "args": {
      "External id": 2481538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612313.657, "dur": 5.156,
    "args": {
      "External id": 2481539,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612315.549, "dur": 2.984,
    "args": {
      "External id": 2481540,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612324.408, "dur": 40.158,
    "args": {
      "External id": 2481541,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612325.426, "dur": 38.614,
    "args": {
      "External id": 2481542,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612368.774, "dur": 15.424,
    "args": {
      "External id": 2481543,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612389.778, "dur": 3.313,
    "args": {
      "External id": 2481544,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612391.839, "dur": 0.514,
    "args": {
      "External id": 2481545,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612397.157, "dur": 50.761,
    "args": {
      "External id": 2481546,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612397.914, "dur": 5.987,
    "args": {
      "External id": 2481547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612400.981, "dur": 2.436,
    "args": {
      "External id": 2481548,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612402.430, "dur": 0.843,
    "args": {
      "External id": 2481549,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612404.584, "dur": 42.811,
    "args": {
      "External id": 2481550,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612405.248, "dur": 41.504,
    "args": {
      "External id": 2481551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612452.582, "dur": 3.800,
    "args": {
      "External id": 2481552,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612454.775, "dur": 0.583,
    "args": {
      "External id": 2481553,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612461.881, "dur": 1.566,
    "args": {
      "External id": 2481554,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612472.801, "dur": 6.033,
    "args": {
      "External id": 2481555,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612474.881, "dur": 3.655,
    "args": {
      "External id": 2481556,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612590.494, "dur": 199.820,
    "args": {
      "External id": 2481557,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612592.831, "dur": 3.810,
    "args": {
      "External id": 2481558,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612600.899, "dur": 188.871,
    "args": {
      "External id": 2481559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612602.204, "dur": 0.380,
    "args": {
      "External id": 2481560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612603.696, "dur": 23.885,
    "args": {
      "External id": 2481561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612629.384, "dur": 5.191,
    "args": {
      "External id": 2481562,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612631.218, "dur": 2.988,
    "args": {
      "External id": 2481563,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612637.495, "dur": 24.776,
    "args": {
      "External id": 2481564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612638.895, "dur": 2.372,
    "args": {
      "External id": 2481565,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612642.578, "dur": 19.407,
    "args": {
      "External id": 2481566,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612645.579, "dur": 2.719,
    "args": {
      "External id": 2481567,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612663.728, "dur": 21.025,
    "args": {
      "External id": 2481568,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612686.334, "dur": 15.583,
    "args": {
      "External id": 2481569,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612704.892, "dur": 15.036,
    "args": {
      "External id": 2481570,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612721.277, "dur": 13.744,
    "args": {
      "External id": 2481571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612736.486, "dur": 24.875,
    "args": {
      "External id": 2481572,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612740.443, "dur": 1.871,
    "args": {
      "External id": 2481573,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612744.183, "dur": 3.072,
    "args": {
      "External id": 2481574,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612762.769, "dur": 13.104,
    "args": {
      "External id": 2481575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612777.347, "dur": 11.378,
    "args": {
      "External id": 2481576,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612797.341, "dur": 1.943,
    "args": {
      "External id": 2481577,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612808.576, "dur": 3.449,
    "args": {
      "External id": 2481578,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612810.617, "dur": 0.462,
    "args": {
      "External id": 2481579,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612884.056, "dur": 88.108,
    "args": {
      "External id": 2481580,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612976.806, "dur": 38.009,
    "args": {
      "External id": 2481581,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656612979.273, "dur": 1.056,
    "args": {
      "External id": 2481582,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613017.541, "dur": 30.917,
    "args": {
      "External id": 2481583,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613054.769, "dur": 8.862,
    "args": {
      "External id": 2481584,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613056.494, "dur": 6.234,
    "args": {
      "External id": 2481585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613061.122, "dur": 1.362,
    "args": {
      "External id": 2481586,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613067.116, "dur": 49.069,
    "args": {
      "External id": 2481587,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613068.088, "dur": 47.433,
    "args": {
      "External id": 2481588,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613120.743, "dur": 17.652,
    "args": {
      "External id": 2481589,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613144.766, "dur": 3.876,
    "args": {
      "External id": 2481590,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613147.003, "dur": 0.751,
    "args": {
      "External id": 2481591,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613152.670, "dur": 50.269,
    "args": {
      "External id": 2481592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613155.886, "dur": 3.790,
    "args": {
      "External id": 2481593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613156.967, "dur": 2.254,
    "args": {
      "External id": 2481594,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613158.114, "dur": 0.973,
    "args": {
      "External id": 2481595,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613160.461, "dur": 42.066,
    "args": {
      "External id": 2481596,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613160.982, "dur": 41.016,
    "args": {
      "External id": 2481597,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613206.738, "dur": 4.153,
    "args": {
      "External id": 2481598,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613208.773, "dur": 1.034,
    "args": {
      "External id": 2481599,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613219.273, "dur": 1.503,
    "args": {
      "External id": 2481600,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613228.174, "dur": 6.422,
    "args": {
      "External id": 2481601,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613230.224, "dur": 4.128,
    "args": {
      "External id": 2481602,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613321.046, "dur": 198.125,
    "args": {
      "External id": 2481603,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613325.864, "dur": 2.689,
    "args": {
      "External id": 2481604,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613332.447, "dur": 186.236,
    "args": {
      "External id": 2481605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613333.657, "dur": 0.787,
    "args": {
      "External id": 2481606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613335.665, "dur": 22.759,
    "args": {
      "External id": 2481607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613360.044, "dur": 5.899,
    "args": {
      "External id": 2481608,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613364.162, "dur": 1.513,
    "args": {
      "External id": 2481609,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613366.864, "dur": 21.995,
    "args": {
      "External id": 2481610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613368.148, "dur": 1.716,
    "args": {
      "External id": 2481611,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613371.006, "dur": 17.563,
    "args": {
      "External id": 2481612,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613373.355, "dur": 2.580,
    "args": {
      "External id": 2481613,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613392.441, "dur": 23.875,
    "args": {
      "External id": 2481614,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613417.713, "dur": 13.933,
    "args": {
      "External id": 2481615,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613433.947, "dur": 15.716,
    "args": {
      "External id": 2481616,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613451.023, "dur": 13.409,
    "args": {
      "External id": 2481617,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613466.396, "dur": 23.054,
    "args": {
      "External id": 2481618,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613470.323, "dur": 1.216,
    "args": {
      "External id": 2481619,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613473.845, "dur": 0.828,
    "args": {
      "External id": 2481620,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613490.689, "dur": 12.415,
    "args": {
      "External id": 2481621,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613506.354, "dur": 11.411,
    "args": {
      "External id": 2481622,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613525.346, "dur": 18.175,
    "args": {
      "External id": 2481623,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613554.782, "dur": 4.587,
    "args": {
      "External id": 2481624,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613557.682, "dur": 0.732,
    "args": {
      "External id": 2481625,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613624.675, "dur": 56.800,
    "args": {
      "External id": 2481626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613686.531, "dur": 4.230,
    "args": {
      "External id": 2481627,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613688.733, "dur": 0.903,
    "args": {
      "External id": 2481628,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613692.128, "dur": 25.146,
    "args": {
      "External id": 2481629,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613723.661, "dur": 6.347,
    "args": {
      "External id": 2481630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613725.579, "dur": 3.894,
    "args": {
      "External id": 2481631,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613727.746, "dur": 1.507,
    "args": {
      "External id": 2481632,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613732.676, "dur": 41.481,
    "args": {
      "External id": 2481633,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613733.793, "dur": 39.804,
    "args": {
      "External id": 2481634,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613777.930, "dur": 16.393,
    "args": {
      "External id": 2481635,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613799.429, "dur": 5.181,
    "args": {
      "External id": 2481636,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613803.215, "dur": 0.635,
    "args": {
      "External id": 2481637,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613808.195, "dur": 49.705,
    "args": {
      "External id": 2481638,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613809.050, "dur": 4.004,
    "args": {
      "External id": 2481639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613810.096, "dur": 2.496,
    "args": {
      "External id": 2481640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613811.681, "dur": 0.774,
    "args": {
      "External id": 2481641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613813.587, "dur": 43.882,
    "args": {
      "External id": 2481642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613816.659, "dur": 40.271,
    "args": {
      "External id": 2481643,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613861.890, "dur": 3.903,
    "args": {
      "External id": 2481644,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613863.914, "dur": 0.885,
    "args": {
      "External id": 2481645,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613871.020, "dur": 1.626,
    "args": {
      "External id": 2481646,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613879.332, "dur": 6.585,
    "args": {
      "External id": 2481647,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613881.435, "dur": 4.237,
    "args": {
      "External id": 2481648,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613966.894, "dur": 276.362,
    "args": {
      "External id": 2481649,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613970.998, "dur": 2.557,
    "args": {
      "External id": 2481650,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613975.007, "dur": 267.442,
    "args": {
      "External id": 2481651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613975.934, "dur": 0.561,
    "args": {
      "External id": 2481652,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656613977.633, "dur": 58.846,
    "args": {
      "External id": 2481653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614038.798, "dur": 5.848,
    "args": {
      "External id": 2481654,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614043.246, "dur": 1.083,
    "args": {
      "External id": 2481655,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614045.549, "dur": 25.159,
    "args": {
      "External id": 2481656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614046.708, "dur": 1.954,
    "args": {
      "External id": 2481657,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614050.007, "dur": 20.436,
    "args": {
      "External id": 2481658,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614055.439, "dur": 2.877,
    "args": {
      "External id": 2481659,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614072.109, "dur": 22.306,
    "args": {
      "External id": 2481660,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614095.592, "dur": 16.962,
    "args": {
      "External id": 2481661,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614115.127, "dur": 14.930,
    "args": {
      "External id": 2481662,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614131.373, "dur": 14.241,
    "args": {
      "External id": 2481663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614147.337, "dur": 64.535,
    "args": {
      "External id": 2481664,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614188.035, "dur": 5.768,
    "args": {
      "External id": 2481665,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614195.652, "dur": 0.535,
    "args": {
      "External id": 2481666,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614215.709, "dur": 13.596,
    "args": {
      "External id": 2481667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614230.278, "dur": 11.339,
    "args": {
      "External id": 2481668,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614251.020, "dur": 2.414,
    "args": {
      "External id": 2481669,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614262.853, "dur": 3.826,
    "args": {
      "External id": 2481670,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614265.507, "dur": 0.375,
    "args": {
      "External id": 2481671,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614337.538, "dur": 57.618,
    "args": {
      "External id": 2481672,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614399.640, "dur": 6.877,
    "args": {
      "External id": 2481673,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614404.457, "dur": 0.987,
    "args": {
      "External id": 2481674,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614408.008, "dur": 24.019,
    "args": {
      "External id": 2481675,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614436.066, "dur": 7.269,
    "args": {
      "External id": 2481676,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614437.566, "dur": 5.110,
    "args": {
      "External id": 2481677,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614439.531, "dur": 2.943,
    "args": {
      "External id": 2481678,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614446.064, "dur": 40.887,
    "args": {
      "External id": 2481679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614447.021, "dur": 39.009,
    "args": {
      "External id": 2481680,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614493.239, "dur": 14.797,
    "args": {
      "External id": 2481681,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614513.281, "dur": 3.140,
    "args": {
      "External id": 2481682,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614514.949, "dur": 0.738,
    "args": {
      "External id": 2481683,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614519.994, "dur": 80.114,
    "args": {
      "External id": 2481684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614520.651, "dur": 5.761,
    "args": {
      "External id": 2481685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614521.181, "dur": 4.733,
    "args": {
      "External id": 2481686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614525.001, "dur": 0.778,
    "args": {
      "External id": 2481687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614541.880, "dur": 57.743,
    "args": {
      "External id": 2481688,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614542.643, "dur": 56.076,
    "args": {
      "External id": 2481689,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614604.751, "dur": 4.345,
    "args": {
      "External id": 2481690,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614607.039, "dur": 0.857,
    "args": {
      "External id": 2481691,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614615.295, "dur": 1.510,
    "args": {
      "External id": 2481692,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614626.312, "dur": 6.670,
    "args": {
      "External id": 2481693,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614628.179, "dur": 4.542,
    "args": {
      "External id": 2481694,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614717.331, "dur": 191.664,
    "args": {
      "External id": 2481695,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614719.340, "dur": 2.066,
    "args": {
      "External id": 2481696,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614724.723, "dur": 183.744,
    "args": {
      "External id": 2481697,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614725.878, "dur": 0.408,
    "args": {
      "External id": 2481698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614727.554, "dur": 21.775,
    "args": {
      "External id": 2481699,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614750.755, "dur": 5.335,
    "args": {
      "External id": 2481700,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614753.184, "dur": 2.460,
    "args": {
      "External id": 2481701,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614759.221, "dur": 22.665,
    "args": {
      "External id": 2481702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614760.147, "dur": 2.140,
    "args": {
      "External id": 2481703,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614763.688, "dur": 17.940,
    "args": {
      "External id": 2481704,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614766.175, "dur": 2.284,
    "args": {
      "External id": 2481705,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614783.228, "dur": 21.253,
    "args": {
      "External id": 2481706,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614805.815, "dur": 14.071,
    "args": {
      "External id": 2481707,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614822.681, "dur": 15.700,
    "args": {
      "External id": 2481708,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614839.485, "dur": 13.263,
    "args": {
      "External id": 2481709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614854.597, "dur": 25.540,
    "args": {
      "External id": 2481710,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614858.169, "dur": 1.955,
    "args": {
      "External id": 2481711,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614862.530, "dur": 2.593,
    "args": {
      "External id": 2481712,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614881.557, "dur": 12.552,
    "args": {
      "External id": 2481713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614895.240, "dur": 12.304,
    "args": {
      "External id": 2481714,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614914.720, "dur": 1.428,
    "args": {
      "External id": 2481715,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614924.338, "dur": 3.038,
    "args": {
      "External id": 2481716,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656614926.347, "dur": 0.327,
    "args": {
      "External id": 2481717,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615022.692, "dur": 60.199,
    "args": {
      "External id": 2481718,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615088.002, "dur": 5.396,
    "args": {
      "External id": 2481719,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615090.984, "dur": 0.987,
    "args": {
      "External id": 2481720,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615094.695, "dur": 26.495,
    "args": {
      "External id": 2481721,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615126.126, "dur": 8.490,
    "args": {
      "External id": 2481722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615127.589, "dur": 6.269,
    "args": {
      "External id": 2481723,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615132.073, "dur": 1.594,
    "args": {
      "External id": 2481724,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615137.097, "dur": 39.302,
    "args": {
      "External id": 2481725,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615138.045, "dur": 37.783,
    "args": {
      "External id": 2481726,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615180.257, "dur": 16.537,
    "args": {
      "External id": 2481727,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615202.310, "dur": 3.365,
    "args": {
      "External id": 2481728,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615204.259, "dur": 0.642,
    "args": {
      "External id": 2481729,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615209.783, "dur": 49.706,
    "args": {
      "External id": 2481730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615212.697, "dur": 3.759,
    "args": {
      "External id": 2481731,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615213.861, "dur": 2.109,
    "args": {
      "External id": 2481732,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615215.161, "dur": 0.633,
    "args": {
      "External id": 2481733,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615217.240, "dur": 41.821,
    "args": {
      "External id": 2481734,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615217.815, "dur": 40.805,
    "args": {
      "External id": 2481735,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615263.287, "dur": 3.555,
    "args": {
      "External id": 2481736,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615265.123, "dur": 0.797,
    "args": {
      "External id": 2481737,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615272.689, "dur": 1.514,
    "args": {
      "External id": 2481738,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615282.634, "dur": 6.895,
    "args": {
      "External id": 2481739,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615284.741, "dur": 4.539,
    "args": {
      "External id": 2481740,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615367.922, "dur": 219.303,
    "args": {
      "External id": 2481741,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615372.821, "dur": 1.940,
    "args": {
      "External id": 2481742,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615375.715, "dur": 210.766,
    "args": {
      "External id": 2481743,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615377.168, "dur": 0.463,
    "args": {
      "External id": 2481744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615381.212, "dur": 21.330,
    "args": {
      "External id": 2481745,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615403.934, "dur": 5.482,
    "args": {
      "External id": 2481746,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615408.384, "dur": 0.773,
    "args": {
      "External id": 2481747,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615419.990, "dur": 22.392,
    "args": {
      "External id": 2481748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615421.186, "dur": 1.898,
    "args": {
      "External id": 2481749,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615424.340, "dur": 17.717,
    "args": {
      "External id": 2481750,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615426.834, "dur": 2.503,
    "args": {
      "External id": 2481751,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615443.884, "dur": 21.671,
    "args": {
      "External id": 2481752,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615466.735, "dur": 14.296,
    "args": {
      "External id": 2481753,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615486.039, "dur": 13.715,
    "args": {
      "External id": 2481754,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615500.997, "dur": 13.592,
    "args": {
      "External id": 2481755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615516.091, "dur": 40.095,
    "args": {
      "External id": 2481756,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615520.271, "dur": 1.064,
    "args": {
      "External id": 2481757,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615523.461, "dur": 0.689,
    "args": {
      "External id": 2481758,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615558.733, "dur": 14.343,
    "args": {
      "External id": 2481759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615574.192, "dur": 11.211,
    "args": {
      "External id": 2481760,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615596.993, "dur": 2.097,
    "args": {
      "External id": 2481761,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615607.045, "dur": 3.835,
    "args": {
      "External id": 2481762,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615609.125, "dur": 0.734,
    "args": {
      "External id": 2481763,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615675.822, "dur": 53.185,
    "args": {
      "External id": 2481764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615733.374, "dur": 4.614,
    "args": {
      "External id": 2481765,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615736.097, "dur": 0.903,
    "args": {
      "External id": 2481766,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615743.649, "dur": 23.969,
    "args": {
      "External id": 2481767,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615774.065, "dur": 9.475,
    "args": {
      "External id": 2481768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615775.430, "dur": 7.489,
    "args": {
      "External id": 2481769,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615779.788, "dur": 2.911,
    "args": {
      "External id": 2481770,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615785.890, "dur": 40.968,
    "args": {
      "External id": 2481771,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615786.886, "dur": 39.459,
    "args": {
      "External id": 2481772,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615830.528, "dur": 15.321,
    "args": {
      "External id": 2481773,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615851.237, "dur": 6.031,
    "args": {
      "External id": 2481774,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615855.613, "dur": 0.889,
    "args": {
      "External id": 2481775,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615860.983, "dur": 49.241,
    "args": {
      "External id": 2481776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615862.063, "dur": 3.376,
    "args": {
      "External id": 2481777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615862.782, "dur": 2.180,
    "args": {
      "External id": 2481778,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615864.039, "dur": 0.648,
    "args": {
      "External id": 2481779,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615866.392, "dur": 43.456,
    "args": {
      "External id": 2481780,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615869.103, "dur": 40.171,
    "args": {
      "External id": 2481781,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615913.813, "dur": 3.766,
    "args": {
      "External id": 2481782,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615915.870, "dur": 0.699,
    "args": {
      "External id": 2481783,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615922.674, "dur": 1.344,
    "args": {
      "External id": 2481784,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615931.147, "dur": 8.565,
    "args": {
      "External id": 2481785,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656615935.368, "dur": 4.079,
    "args": {
      "External id": 2481786,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616076.119, "dur": 202.402,
    "args": {
      "External id": 2481787,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616078.798, "dur": 3.982,
    "args": {
      "External id": 2481788,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616086.705, "dur": 191.367,
    "args": {
      "External id": 2481789,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616088.216, "dur": 0.349,
    "args": {
      "External id": 2481790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616089.694, "dur": 24.163,
    "args": {
      "External id": 2481791,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616115.857, "dur": 5.467,
    "args": {
      "External id": 2481792,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616118.093, "dur": 2.924,
    "args": {
      "External id": 2481793,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616122.460, "dur": 24.732,
    "args": {
      "External id": 2481794,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616125.887, "dur": 1.410,
    "args": {
      "External id": 2481795,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616128.528, "dur": 18.387,
    "args": {
      "External id": 2481796,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616131.236, "dur": 2.757,
    "args": {
      "External id": 2481797,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616148.714, "dur": 25.152,
    "args": {
      "External id": 2481798,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616175.369, "dur": 14.055,
    "args": {
      "External id": 2481799,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616192.401, "dur": 15.700,
    "args": {
      "External id": 2481800,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616209.790, "dur": 13.514,
    "args": {
      "External id": 2481801,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616224.977, "dur": 24.435,
    "args": {
      "External id": 2481802,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616226.957, "dur": 1.263,
    "args": {
      "External id": 2481803,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616232.152, "dur": 2.735,
    "args": {
      "External id": 2481804,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616250.738, "dur": 12.889,
    "args": {
      "External id": 2481805,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616264.787, "dur": 12.328,
    "args": {
      "External id": 2481806,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616285.222, "dur": 2.263,
    "args": {
      "External id": 2481807,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616296.104, "dur": 3.440,
    "args": {
      "External id": 2481808,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616298.280, "dur": 0.401,
    "args": {
      "External id": 2481809,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616362.852, "dur": 62.220,
    "args": {
      "External id": 2481810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616429.838, "dur": 6.763,
    "args": {
      "External id": 2481811,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616434.586, "dur": 0.821,
    "args": {
      "External id": 2481812,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616438.148, "dur": 26.389,
    "args": {
      "External id": 2481813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616469.301, "dur": 5.265,
    "args": {
      "External id": 2481814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616470.762, "dur": 3.224,
    "args": {
      "External id": 2481815,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616472.659, "dur": 1.143,
    "args": {
      "External id": 2481816,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616477.051, "dur": 42.132,
    "args": {
      "External id": 2481817,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616479.932, "dur": 38.647,
    "args": {
      "External id": 2481818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616523.172, "dur": 32.810,
    "args": {
      "External id": 2481819,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616562.580, "dur": 27.507,
    "args": {
      "External id": 2481820,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616565.208, "dur": 24.525,
    "args": {
      "External id": 2481821,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616570.940, "dur": 1.178,
    "args": {
      "External id": 2481822,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616595.559, "dur": 27.220,
    "args": {
      "External id": 2481823,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616597.037, "dur": 25.540,
    "args": {
      "External id": 2481824,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616601.961, "dur": 3.984,
    "args": {
      "External id": 2481825,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616606.958, "dur": 15.165,
    "args": {
      "External id": 2481826,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616636.905, "dur": 6.958,
    "args": {
      "External id": 2481827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616640.641, "dur": 2.869,
    "args": {
      "External id": 2481828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616644.810, "dur": 1.222,
    "args": {
      "External id": 2481829,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616645.458, "dur": 0.474,
    "args": {
      "External id": 2481830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616685.801, "dur": 22.623,
    "args": {
      "External id": 2481831,"Sequence number": 24740632, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616710.449, "dur": 14.282,
    "args": {
      "External id": 2481832,"Sequence number": 24740633, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9535
    }
  },
  {
    "ph": "s", "id": 16, "pid": 1336760, "tid": 1336760, "ts": 1514656616710.449,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616730.620, "dur": 6.147,
    "args": {
      "External id": 2481833,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616734.078, "dur": 1.286,
    "args": {
      "External id": 2481834,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616741.679, "dur": 6.525,
    "args": {
      "External id": 2481835,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616745.927, "dur": 1.019,
    "args": {
      "External id": 2481836,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616749.569, "dur": 2.649,
    "args": {
      "External id": 2481837,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616751.024, "dur": 0.636,
    "args": {
      "External id": 2481838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616755.873, "dur": 7.959,
    "args": {
      "External id": 2481839,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9542
    }
  },
  {
    "ph": "s", "id": 15, "pid": 1336760, "tid": 1336760, "ts": 1514656616755.873,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616761.508, "dur": 1.152,
    "args": {
      "External id": 2481840,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616765.170, "dur": 6.007,
    "args": {
      "External id": 2481841,"Sequence number": 24740635, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9544
    }
  },
  {
    "ph": "s", "id": 14, "pid": 1336760, "tid": 1336760, "ts": 1514656616765.170,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616769.486, "dur": 0.936,
    "args": {
      "External id": 2481842,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616772.427, "dur": 7.267,
    "args": {
      "External id": 2481843,"Sequence number": 24740636, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9546
    }
  },
  {
    "ph": "s", "id": 13, "pid": 1336760, "tid": 1336760, "ts": 1514656616772.427,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616777.825, "dur": 0.788,
    "args": {
      "External id": 2481844,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616780.804, "dur": 5.658,
    "args": {
      "External id": 2481845,"Sequence number": 24740637, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9548
    }
  },
  {
    "ph": "s", "id": 12, "pid": 1336760, "tid": 1336760, "ts": 1514656616780.804,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616784.360, "dur": 1.185,
    "args": {
      "External id": 2481846,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616790.182, "dur": 30.362,
    "args": {
      "External id": 2481847,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616791.867, "dur": 28.468,
    "args": {
      "External id": 2481848,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616794.373, "dur": 6.352,
    "args": {
      "External id": 2481849,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616796.847, "dur": 3.337,
    "args": {
      "External id": 2481850,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616801.527, "dur": 18.377,
    "args": {
      "External id": 2481851,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616846.659, "dur": 6.507,
    "args": {
      "External id": 2481852,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9555
    }
  },
  {
    "ph": "s", "id": 11, "pid": 1336760, "tid": 1336760, "ts": 1514656616846.659,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616855.251, "dur": 1.203,
    "args": {
      "External id": 2481853,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616889.799, "dur": 43845.400,
    "args": {
      "External id": 2481854,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9557
    }
  },
  {
    "ph": "s", "id": 10, "pid": 1336760, "tid": 1336760, "ts": 1514656616889.799,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616904.079, "dur": 26.007,
    "args": {
      "External id": 2481855,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616904.955, "dur": 24.946,
    "args": {
      "External id": 2481856,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616906.456, "dur": 6.076,
    "args": {
      "External id": 2481857,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616907.897, "dur": 4.284,
    "args": {
      "External id": 2481858,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616913.098, "dur": 16.371,
    "args": {
      "External id": 2481859,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616944.835, "dur": 25.242,
    "args": {
      "External id": 2481860,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616945.939, "dur": 6.137,
    "args": {
      "External id": 2481861,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616947.901, "dur": 3.849,
    "args": {
      "External id": 2481862,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616953.280, "dur": 16.584,
    "args": {
      "External id": 2481863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616955.314, "dur": 14.210,
    "args": {
      "External id": 2481864,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616973.623, "dur": 60.407,
    "args": {
      "External id": 2481865,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616974.960, "dur": 7.394,
    "args": {
      "External id": 2481866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616978.521, "dur": 3.610,
    "args": {
      "External id": 2481867,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616982.894, "dur": 50.936,
    "args": {
      "External id": 2481868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656616983.452, "dur": 49.743,
    "args": {
      "External id": 2481869,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617042.148, "dur": 21.107,
    "args": {
      "External id": 2481870,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617046.100, "dur": 3.765,
    "args": {
      "External id": 2481871,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617050.585, "dur": 12.424,
    "args": {
      "External id": 2481872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617051.663, "dur": 11.064,
    "args": {
      "External id": 2481873,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617070.009, "dur": 25.512,
    "args": {
      "External id": 2481874,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617098.163, "dur": 51.895,
    "args": {
      "External id": 2481875,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617100.349, "dur": 49.264,
    "args": {
      "External id": 2481876,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617105.890, "dur": 1.054,
    "args": {
      "External id": 2481877,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617108.342, "dur": 24.657,
    "args": {
      "External id": 2481878,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617109.505, "dur": 23.310,
    "args": {
      "External id": 2481879,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617114.231, "dur": 3.734,
    "args": {
      "External id": 2481880,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617119.061, "dur": 13.425,
    "args": {
      "External id": 2481881,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617153.641, "dur": 37755.984,
    "args": {
      "External id": 2481882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514656617154.959, "dur": 37753.292,
    "args": {
      "External id": 2481883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654921.283, "dur": 9.127,
    "args": {
      "External id": 2481884,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654927.398, "dur": 1.192,
    "args": {
      "External id": 2481885,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654935.525, "dur": 130.342,
    "args": {
      "External id": 2481886,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654936.914, "dur": 6.993,
    "args": {
      "External id": 2481887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654939.264, "dur": 3.772,
    "args": {
      "External id": 2481888,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654941.444, "dur": 1.323,
    "args": {
      "External id": 2481889,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654945.107, "dur": 119.859,
    "args": {
      "External id": 2481890,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656654949.186, "dur": 114.683,
    "args": {
      "External id": 2481891,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655071.403, "dur": 4.991,
    "args": {
      "External id": 2481892,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655074.102, "dur": 0.750,
    "args": {
      "External id": 2481893,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655084.355, "dur": 2.749,
    "args": {
      "External id": 2481894,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655096.350, "dur": 7.228,
    "args": {
      "External id": 2481895,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655098.452, "dur": 4.840,
    "args": {
      "External id": 2481896,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655235.137, "dur": 216.209,
    "args": {
      "External id": 2481897,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655237.747, "dur": 2.762,
    "args": {
      "External id": 2481898,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655244.647, "dur": 206.132,
    "args": {
      "External id": 2481899,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655246.215, "dur": 0.416,
    "args": {
      "External id": 2481900,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655248.194, "dur": 28.274,
    "args": {
      "External id": 2481901,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655278.373, "dur": 5.649,
    "args": {
      "External id": 2481902,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655280.977, "dur": 2.648,
    "args": {
      "External id": 2481903,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655285.353, "dur": 27.340,
    "args": {
      "External id": 2481904,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655286.734, "dur": 1.458,
    "args": {
      "External id": 2481905,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655291.823, "dur": 20.494,
    "args": {
      "External id": 2481906,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655295.760, "dur": 3.307,
    "args": {
      "External id": 2481907,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655314.443, "dur": 24.817,
    "args": {
      "External id": 2481908,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655341.145, "dur": 16.101,
    "args": {
      "External id": 2481909,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655360.532, "dur": 17.252,
    "args": {
      "External id": 2481910,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655379.529, "dur": 13.986,
    "args": {
      "External id": 2481911,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655395.397, "dur": 24.557,
    "args": {
      "External id": 2481912,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655397.357, "dur": 2.044,
    "args": {
      "External id": 2481913,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655403.826, "dur": 1.089,
    "args": {
      "External id": 2481914,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655421.557, "dur": 13.370,
    "args": {
      "External id": 2481915,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655436.472, "dur": 13.112,
    "args": {
      "External id": 2481916,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655458.872, "dur": 2.044,
    "args": {
      "External id": 2481917,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655467.593, "dur": 4.420,
    "args": {
      "External id": 2481918,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655470.050, "dur": 0.958,
    "args": {
      "External id": 2481919,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655559.697, "dur": 71.267,
    "args": {
      "External id": 2481920,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655637.111, "dur": 10.761,
    "args": {
      "External id": 2481921,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655642.169, "dur": 2.788,
    "args": {
      "External id": 2481922,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655649.738, "dur": 29.890,
    "args": {
      "External id": 2481923,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655685.532, "dur": 7.010,
    "args": {
      "External id": 2481924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655687.521, "dur": 4.169,
    "args": {
      "External id": 2481925,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655689.955, "dur": 1.290,
    "args": {
      "External id": 2481926,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655695.329, "dur": 47.035,
    "args": {
      "External id": 2481927,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655699.055, "dur": 42.568,
    "args": {
      "External id": 2481928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655746.931, "dur": 18.180,
    "args": {
      "External id": 2481929,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655771.651, "dur": 4.073,
    "args": {
      "External id": 2481930,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655773.921, "dur": 0.698,
    "args": {
      "External id": 2481931,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655780.142, "dur": 51.355,
    "args": {
      "External id": 2481932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655781.079, "dur": 6.169,
    "args": {
      "External id": 2481933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655781.942, "dur": 4.569,
    "args": {
      "External id": 2481934,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655785.676, "dur": 0.614,
    "args": {
      "External id": 2481935,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655788.050, "dur": 42.922,
    "args": {
      "External id": 2481936,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655788.522, "dur": 41.969,
    "args": {
      "External id": 2481937,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655835.588, "dur": 3.377,
    "args": {
      "External id": 2481938,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655837.214, "dur": 0.800,
    "args": {
      "External id": 2481939,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655845.592, "dur": 1.573,
    "args": {
      "External id": 2481940,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655856.762, "dur": 6.760,
    "args": {
      "External id": 2481941,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655858.964, "dur": 4.289,
    "args": {
      "External id": 2481942,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655958.373, "dur": 237.308,
    "args": {
      "External id": 2481943,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655960.563, "dur": 4.149,
    "args": {
      "External id": 2481944,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655965.889, "dur": 229.165,
    "args": {
      "External id": 2481945,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655967.165, "dur": 0.536,
    "args": {
      "External id": 2481946,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656655970.998, "dur": 57.454,
    "args": {
      "External id": 2481947,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656031.124, "dur": 5.964,
    "args": {
      "External id": 2481948,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656035.909, "dur": 0.878,
    "args": {
      "External id": 2481949,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656038.127, "dur": 24.949,
    "args": {
      "External id": 2481950,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656038.976, "dur": 2.081,
    "args": {
      "External id": 2481951,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656042.253, "dur": 20.579,
    "args": {
      "External id": 2481952,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656045.911, "dur": 2.830,
    "args": {
      "External id": 2481953,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656064.659, "dur": 23.262,
    "args": {
      "External id": 2481954,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656089.476, "dur": 14.999,
    "args": {
      "External id": 2481955,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656107.735, "dur": 16.105,
    "args": {
      "External id": 2481956,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656127.188, "dur": 13.727,
    "args": {
      "External id": 2481957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656142.708, "dur": 23.382,
    "args": {
      "External id": 2481958,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656144.839, "dur": 1.372,
    "args": {
      "External id": 2481959,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656148.102, "dur": 2.515,
    "args": {
      "External id": 2481960,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656167.606, "dur": 13.827,
    "args": {
      "External id": 2481961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656182.743, "dur": 11.209,
    "args": {
      "External id": 2481962,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656204.139, "dur": 2.463,
    "args": {
      "External id": 2481963,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656217.856, "dur": 3.563,
    "args": {
      "External id": 2481964,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656219.926, "dur": 0.529,
    "args": {
      "External id": 2481965,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656294.076, "dur": 58.821,
    "args": {
      "External id": 2481966,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656357.874, "dur": 4.464,
    "args": {
      "External id": 2481967,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656360.250, "dur": 1.021,
    "args": {
      "External id": 2481968,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656363.869, "dur": 25.831,
    "args": {
      "External id": 2481969,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656394.363, "dur": 8.256,
    "args": {
      "External id": 2481970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656396.265, "dur": 5.627,
    "args": {
      "External id": 2481971,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656400.347, "dur": 1.341,
    "args": {
      "External id": 2481972,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656405.171, "dur": 41.905,
    "args": {
      "External id": 2481973,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656406.373, "dur": 40.055,
    "args": {
      "External id": 2481974,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656451.008, "dur": 16.248,
    "args": {
      "External id": 2481975,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656472.916, "dur": 4.148,
    "args": {
      "External id": 2481976,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656475.388, "dur": 0.752,
    "args": {
      "External id": 2481977,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656483.253, "dur": 62.486,
    "args": {
      "External id": 2481978,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656484.013, "dur": 3.472,
    "args": {
      "External id": 2481979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656484.886, "dur": 2.083,
    "args": {
      "External id": 2481980,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656485.989, "dur": 0.859,
    "args": {
      "External id": 2481981,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656488.242, "dur": 56.879,
    "args": {
      "External id": 2481982,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656488.935, "dur": 55.039,
    "args": {
      "External id": 2481983,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656569.462, "dur": 4.830,
    "args": {
      "External id": 2481984,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656572.188, "dur": 0.920,
    "args": {
      "External id": 2481985,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656582.919, "dur": 1.868,
    "args": {
      "External id": 2481986,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656592.729, "dur": 8.507,
    "args": {
      "External id": 2481987,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656594.393, "dur": 6.601,
    "args": {
      "External id": 2481988,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656691.747, "dur": 198.151,
    "args": {
      "External id": 2481989,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656693.718, "dur": 2.523,
    "args": {
      "External id": 2481990,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656697.838, "dur": 191.530,
    "args": {
      "External id": 2481991,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656701.346, "dur": 0.332,
    "args": {
      "External id": 2481992,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656702.729, "dur": 26.571,
    "args": {
      "External id": 2481993,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656731.151, "dur": 2.980,
    "args": {
      "External id": 2481994,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656733.119, "dur": 0.771,
    "args": {
      "External id": 2481995,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656735.363, "dur": 24.631,
    "args": {
      "External id": 2481996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656736.789, "dur": 1.914,
    "args": {
      "External id": 2481997,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656739.838, "dur": 19.847,
    "args": {
      "External id": 2481998,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656744.654, "dur": 2.668,
    "args": {
      "External id": 2481999,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656761.272, "dur": 25.268,
    "args": {
      "External id": 2482000,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656788.230, "dur": 13.825,
    "args": {
      "External id": 2482001,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656804.416, "dur": 15.372,
    "args": {
      "External id": 2482002,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656823.559, "dur": 12.894,
    "args": {
      "External id": 2482003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656838.110, "dur": 20.747,
    "args": {
      "External id": 2482004,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656839.818, "dur": 1.900,
    "args": {
      "External id": 2482005,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656843.512, "dur": 0.766,
    "args": {
      "External id": 2482006,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656860.570, "dur": 12.879,
    "args": {
      "External id": 2482007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656874.655, "dur": 12.090,
    "args": {
      "External id": 2482008,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656896.075, "dur": 1.456,
    "args": {
      "External id": 2482009,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656905.601, "dur": 2.998,
    "args": {
      "External id": 2482010,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656907.505, "dur": 0.395,
    "args": {
      "External id": 2482011,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656656973.768, "dur": 118.558,
    "args": {
      "External id": 2482012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657099.656, "dur": 5.803,
    "args": {
      "External id": 2482013,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657102.615, "dur": 1.243,
    "args": {
      "External id": 2482014,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657107.089, "dur": 27.649,
    "args": {
      "External id": 2482015,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657140.189, "dur": 8.210,
    "args": {
      "External id": 2482016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657144.256, "dur": 3.551,
    "args": {
      "External id": 2482017,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657146.531, "dur": 1.058,
    "args": {
      "External id": 2482018,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657154.803, "dur": 48.244,
    "args": {
      "External id": 2482019,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657155.913, "dur": 46.576,
    "args": {
      "External id": 2482020,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657206.884, "dur": 18.055,
    "args": {
      "External id": 2482021,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657230.630, "dur": 6.183,
    "args": {
      "External id": 2482022,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657235.228, "dur": 0.802,
    "args": {
      "External id": 2482023,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657241.172, "dur": 50.105,
    "args": {
      "External id": 2482024,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657242.104, "dur": 5.287,
    "args": {
      "External id": 2482025,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657242.831, "dur": 4.017,
    "args": {
      "External id": 2482026,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657243.938, "dur": 2.707,
    "args": {
      "External id": 2482027,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657247.832, "dur": 42.715,
    "args": {
      "External id": 2482028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657248.465, "dur": 41.372,
    "args": {
      "External id": 2482029,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657297.168, "dur": 3.854,
    "args": {
      "External id": 2482030,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657299.202, "dur": 0.872,
    "args": {
      "External id": 2482031,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657306.928, "dur": 1.498,
    "args": {
      "External id": 2482032,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657315.318, "dur": 6.533,
    "args": {
      "External id": 2482033,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657317.298, "dur": 4.320,
    "args": {
      "External id": 2482034,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657409.689, "dur": 207.110,
    "args": {
      "External id": 2482035,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657411.892, "dur": 2.374,
    "args": {
      "External id": 2482036,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657415.663, "dur": 200.590,
    "args": {
      "External id": 2482037,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657416.970, "dur": 0.500,
    "args": {
      "External id": 2482038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657420.395, "dur": 22.048,
    "args": {
      "External id": 2482039,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657444.102, "dur": 3.153,
    "args": {
      "External id": 2482040,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657446.199, "dur": 0.855,
    "args": {
      "External id": 2482041,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657448.061, "dur": 26.639,
    "args": {
      "External id": 2482042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657449.475, "dur": 3.526,
    "args": {
      "External id": 2482043,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657456.017, "dur": 18.310,
    "args": {
      "External id": 2482044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657458.468, "dur": 2.433,
    "args": {
      "External id": 2482045,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657476.108, "dur": 20.678,
    "args": {
      "External id": 2482046,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657498.386, "dur": 14.753,
    "args": {
      "External id": 2482047,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657516.172, "dur": 30.159,
    "args": {
      "External id": 2482048,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657548.408, "dur": 14.396,
    "args": {
      "External id": 2482049,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657564.450, "dur": 22.166,
    "args": {
      "External id": 2482050,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657566.568, "dur": 1.677,
    "args": {
      "External id": 2482051,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657570.703, "dur": 1.035,
    "args": {
      "External id": 2482052,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657590.184, "dur": 12.616,
    "args": {
      "External id": 2482053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657603.774, "dur": 11.421,
    "args": {
      "External id": 2482054,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657624.107, "dur": 1.924,
    "args": {
      "External id": 2482055,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657634.610, "dur": 3.429,
    "args": {
      "External id": 2482056,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657637.008, "dur": 0.344,
    "args": {
      "External id": 2482057,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657703.402, "dur": 53.598,
    "args": {
      "External id": 2482058,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657761.588, "dur": 6.498,
    "args": {
      "External id": 2482059,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657766.176, "dur": 0.972,
    "args": {
      "External id": 2482060,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657769.639, "dur": 23.759,
    "args": {
      "External id": 2482061,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657797.726, "dur": 7.178,
    "args": {
      "External id": 2482062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657798.790, "dur": 5.511,
    "args": {
      "External id": 2482063,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657800.678, "dur": 3.416,
    "args": {
      "External id": 2482064,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657807.040, "dur": 42.565,
    "args": {
      "External id": 2482065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657808.021, "dur": 40.975,
    "args": {
      "External id": 2482066,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657855.387, "dur": 15.232,
    "args": {
      "External id": 2482067,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657876.285, "dur": 3.794,
    "args": {
      "External id": 2482068,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657878.377, "dur": 0.969,
    "args": {
      "External id": 2482069,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657883.516, "dur": 48.151,
    "args": {
      "External id": 2482070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657884.259, "dur": 5.236,
    "args": {
      "External id": 2482071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657884.870, "dur": 4.097,
    "args": {
      "External id": 2482072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657888.043, "dur": 0.784,
    "args": {
      "External id": 2482073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657890.048, "dur": 41.175,
    "args": {
      "External id": 2482074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657890.819, "dur": 39.912,
    "args": {
      "External id": 2482075,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657943.292, "dur": 4.546,
    "args": {
      "External id": 2482076,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657945.842, "dur": 0.848,
    "args": {
      "External id": 2482077,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657953.045, "dur": 1.469,
    "args": {
      "External id": 2482078,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657961.543, "dur": 9.507,
    "args": {
      "External id": 2482079,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656657965.210, "dur": 5.587,
    "args": {
      "External id": 2482080,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658089.321, "dur": 195.169,
    "args": {
      "External id": 2482081,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658091.582, "dur": 3.032,
    "args": {
      "External id": 2482082,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658096.213, "dur": 187.851,
    "args": {
      "External id": 2482083,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658097.527, "dur": 0.340,
    "args": {
      "External id": 2482084,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658098.831, "dur": 26.121,
    "args": {
      "External id": 2482085,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658126.497, "dur": 3.247,
    "args": {
      "External id": 2482086,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658128.591, "dur": 0.881,
    "args": {
      "External id": 2482087,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658130.564, "dur": 27.123,
    "args": {
      "External id": 2482088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658134.106, "dur": 2.047,
    "args": {
      "External id": 2482089,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658137.172, "dur": 20.204,
    "args": {
      "External id": 2482090,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658142.035, "dur": 2.955,
    "args": {
      "External id": 2482091,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658159.190, "dur": 22.875,
    "args": {
      "External id": 2482092,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658183.435, "dur": 13.656,
    "args": {
      "External id": 2482093,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658200.006, "dur": 16.544,
    "args": {
      "External id": 2482094,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658217.685, "dur": 13.241,
    "args": {
      "External id": 2482095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658232.418, "dur": 22.238,
    "args": {
      "External id": 2482096,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658234.328, "dur": 1.698,
    "args": {
      "External id": 2482097,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658239.825, "dur": 0.831,
    "args": {
      "External id": 2482098,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658255.856, "dur": 13.194,
    "args": {
      "External id": 2482099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658270.051, "dur": 13.069,
    "args": {
      "External id": 2482100,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658290.993, "dur": 2.093,
    "args": {
      "External id": 2482101,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658301.117, "dur": 3.527,
    "args": {
      "External id": 2482102,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658303.306, "dur": 0.620,
    "args": {
      "External id": 2482103,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658367.685, "dur": 58.384,
    "args": {
      "External id": 2482104,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658430.738, "dur": 6.574,
    "args": {
      "External id": 2482105,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658435.052, "dur": 1.209,
    "args": {
      "External id": 2482106,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658438.576, "dur": 25.493,
    "args": {
      "External id": 2482107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658468.745, "dur": 5.540,
    "args": {
      "External id": 2482108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658470.405, "dur": 3.096,
    "args": {
      "External id": 2482109,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658472.172, "dur": 1.123,
    "args": {
      "External id": 2482110,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658476.715, "dur": 44.212,
    "args": {
      "External id": 2482111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658480.671, "dur": 39.604,
    "args": {
      "External id": 2482112,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658524.538, "dur": 34.274,
    "args": {
      "External id": 2482113,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658566.900, "dur": 4.730,
    "args": {
      "External id": 2482114,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658569.820, "dur": 0.795,
    "args": {
      "External id": 2482115,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658575.854, "dur": 67.731,
    "args": {
      "External id": 2482116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658585.969, "dur": 8.496,
    "args": {
      "External id": 2482117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658589.344, "dur": 4.506,
    "args": {
      "External id": 2482118,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658590.674, "dur": 2.991,
    "args": {
      "External id": 2482119,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658595.162, "dur": 47.921,
    "args": {
      "External id": 2482120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658595.763, "dur": 46.451,
    "args": {
      "External id": 2482121,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658647.779, "dur": 4.057,
    "args": {
      "External id": 2482122,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658649.909, "dur": 0.905,
    "args": {
      "External id": 2482123,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658659.409, "dur": 1.380,
    "args": {
      "External id": 2482124,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658668.495, "dur": 5.669,
    "args": {
      "External id": 2482125,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658670.241, "dur": 3.671,
    "args": {
      "External id": 2482126,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658764.948, "dur": 187.480,
    "args": {
      "External id": 2482127,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658767.011, "dur": 2.420,
    "args": {
      "External id": 2482128,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658770.782, "dur": 181.199,
    "args": {
      "External id": 2482129,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658773.976, "dur": 0.287,
    "args": {
      "External id": 2482130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658777.487, "dur": 21.262,
    "args": {
      "External id": 2482131,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658800.548, "dur": 2.775,
    "args": {
      "External id": 2482132,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658802.267, "dur": 0.848,
    "args": {
      "External id": 2482133,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658804.096, "dur": 23.498,
    "args": {
      "External id": 2482134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658805.371, "dur": 3.477,
    "args": {
      "External id": 2482135,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658810.060, "dur": 17.227,
    "args": {
      "External id": 2482136,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658812.361, "dur": 2.804,
    "args": {
      "External id": 2482137,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658829.113, "dur": 20.360,
    "args": {
      "External id": 2482138,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658850.950, "dur": 14.721,
    "args": {
      "External id": 2482139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658868.233, "dur": 13.379,
    "args": {
      "External id": 2482140,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658885.645, "dur": 14.279,
    "args": {
      "External id": 2482141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658901.475, "dur": 21.133,
    "args": {
      "External id": 2482142,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658903.306, "dur": 1.904,
    "args": {
      "External id": 2482143,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658907.082, "dur": 0.854,
    "args": {
      "External id": 2482144,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658923.900, "dur": 12.703,
    "args": {
      "External id": 2482145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658937.497, "dur": 11.304,
    "args": {
      "External id": 2482146,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658958.446, "dur": 1.528,
    "args": {
      "External id": 2482147,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658968.303, "dur": 3.060,
    "args": {
      "External id": 2482148,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656658970.131, "dur": 0.478,
    "args": {
      "External id": 2482149,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659085.271, "dur": 59.154,
    "args": {
      "External id": 2482150,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659150.167, "dur": 5.831,
    "args": {
      "External id": 2482151,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659153.218, "dur": 1.345,
    "args": {
      "External id": 2482152,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659157.578, "dur": 25.176,
    "args": {
      "External id": 2482153,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659187.770, "dur": 10.338,
    "args": {
      "External id": 2482154,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659191.564, "dur": 5.920,
    "args": {
      "External id": 2482155,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659193.324, "dur": 3.921,
    "args": {
      "External id": 2482156,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659200.854, "dur": 42.492,
    "args": {
      "External id": 2482157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659202.034, "dur": 40.485,
    "args": {
      "External id": 2482158,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659247.064, "dur": 16.104,
    "args": {
      "External id": 2482159,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659268.428, "dur": 6.073,
    "args": {
      "External id": 2482160,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659272.985, "dur": 0.707,
    "args": {
      "External id": 2482161,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659278.453, "dur": 46.187,
    "args": {
      "External id": 2482162,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659279.472, "dur": 3.011,
    "args": {
      "External id": 2482163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659280.110, "dur": 1.871,
    "args": {
      "External id": 2482164,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659281.117, "dur": 0.540,
    "args": {
      "External id": 2482165,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659282.970, "dur": 41.246,
    "args": {
      "External id": 2482166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659283.653, "dur": 39.923,
    "args": {
      "External id": 2482167,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659328.447, "dur": 4.413,
    "args": {
      "External id": 2482168,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659330.689, "dur": 1.023,
    "args": {
      "External id": 2482169,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659340.999, "dur": 1.596,
    "args": {
      "External id": 2482170,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659350.127, "dur": 6.838,
    "args": {
      "External id": 2482171,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659352.095, "dur": 4.588,
    "args": {
      "External id": 2482172,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659435.644, "dur": 249.359,
    "args": {
      "External id": 2482173,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659437.500, "dur": 2.096,
    "args": {
      "External id": 2482174,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659445.590, "dur": 238.649,
    "args": {
      "External id": 2482175,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659447.121, "dur": 0.582,
    "args": {
      "External id": 2482176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659482.951, "dur": 22.459,
    "args": {
      "External id": 2482177,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659507.487, "dur": 4.328,
    "args": {
      "External id": 2482178,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659509.264, "dur": 2.300,
    "args": {
      "External id": 2482179,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659512.602, "dur": 39.095,
    "args": {
      "External id": 2482180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659513.770, "dur": 1.623,
    "args": {
      "External id": 2482181,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659516.356, "dur": 34.609,
    "args": {
      "External id": 2482182,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659518.598, "dur": 2.771,
    "args": {
      "External id": 2482183,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659556.110, "dur": 24.347,
    "args": {
      "External id": 2482184,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659582.080, "dur": 13.886,
    "args": {
      "External id": 2482185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659598.982, "dur": 15.742,
    "args": {
      "External id": 2482186,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659615.833, "dur": 13.280,
    "args": {
      "External id": 2482187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659630.674, "dur": 22.922,
    "args": {
      "External id": 2482188,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659632.469, "dur": 1.632,
    "args": {
      "External id": 2482189,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659635.946, "dur": 2.412,
    "args": {
      "External id": 2482190,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659655.213, "dur": 12.853,
    "args": {
      "External id": 2482191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659671.239, "dur": 11.960,
    "args": {
      "External id": 2482192,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659692.949, "dur": 2.249,
    "args": {
      "External id": 2482193,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659703.437, "dur": 3.414,
    "args": {
      "External id": 2482194,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659705.631, "dur": 0.351,
    "args": {
      "External id": 2482195,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659771.798, "dur": 57.267,
    "args": {
      "External id": 2482196,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659833.583, "dur": 4.681,
    "args": {
      "External id": 2482197,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659836.161, "dur": 1.063,
    "args": {
      "External id": 2482198,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659839.517, "dur": 25.336,
    "args": {
      "External id": 2482199,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659871.749, "dur": 5.553,
    "args": {
      "External id": 2482200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659873.410, "dur": 3.325,
    "args": {
      "External id": 2482201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659875.147, "dur": 1.417,
    "args": {
      "External id": 2482202,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659879.759, "dur": 40.608,
    "args": {
      "External id": 2482203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659880.609, "dur": 39.038,
    "args": {
      "External id": 2482204,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659923.897, "dur": 15.180,
    "args": {
      "External id": 2482205,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659944.162, "dur": 5.959,
    "args": {
      "External id": 2482206,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659948.646, "dur": 0.609,
    "args": {
      "External id": 2482207,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659953.637, "dur": 87.385,
    "args": {
      "External id": 2482208,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659954.851, "dur": 3.079,
    "args": {
      "External id": 2482209,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659955.345, "dur": 2.100,
    "args": {
      "External id": 2482210,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659956.532, "dur": 0.805,
    "args": {
      "External id": 2482211,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659958.408, "dur": 82.130,
    "args": {
      "External id": 2482212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656659960.951, "dur": 78.560,
    "args": {
      "External id": 2482213,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660046.763, "dur": 4.861,
    "args": {
      "External id": 2482214,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660049.326, "dur": 1.048,
    "args": {
      "External id": 2482215,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660057.500, "dur": 1.484,
    "args": {
      "External id": 2482216,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660066.147, "dur": 8.606,
    "args": {
      "External id": 2482217,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660070.123, "dur": 4.376,
    "args": {
      "External id": 2482218,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660157.389, "dur": 198.063,
    "args": {
      "External id": 2482219,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660161.841, "dur": 2.299,
    "args": {
      "External id": 2482220,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660166.002, "dur": 189.022,
    "args": {
      "External id": 2482221,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660166.931, "dur": 0.548,
    "args": {
      "External id": 2482222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660168.514, "dur": 23.686,
    "args": {
      "External id": 2482223,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660193.653, "dur": 4.910,
    "args": {
      "External id": 2482224,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660197.596, "dur": 0.730,
    "args": {
      "External id": 2482225,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660199.412, "dur": 29.010,
    "args": {
      "External id": 2482226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660207.797, "dur": 1.596,
    "args": {
      "External id": 2482227,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660210.902, "dur": 17.235,
    "args": {
      "External id": 2482228,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660213.490, "dur": 2.475,
    "args": {
      "External id": 2482229,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660229.764, "dur": 23.483,
    "args": {
      "External id": 2482230,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660254.670, "dur": 17.643,
    "args": {
      "External id": 2482231,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660275.367, "dur": 14.053,
    "args": {
      "External id": 2482232,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660290.672, "dur": 13.775,
    "args": {
      "External id": 2482233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660305.764, "dur": 21.356,
    "args": {
      "External id": 2482234,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660307.681, "dur": 1.392,
    "args": {
      "External id": 2482235,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660312.519, "dur": 0.835,
    "args": {
      "External id": 2482236,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660328.599, "dur": 13.007,
    "args": {
      "External id": 2482237,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660342.546, "dur": 11.528,
    "args": {
      "External id": 2482238,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660361.468, "dur": 1.504,
    "args": {
      "External id": 2482239,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660370.711, "dur": 3.194,
    "args": {
      "External id": 2482240,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660372.558, "dur": 0.367,
    "args": {
      "External id": 2482241,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660434.003, "dur": 52.363,
    "args": {
      "External id": 2482242,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660491.049, "dur": 5.936,
    "args": {
      "External id": 2482243,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660495.366, "dur": 0.713,
    "args": {
      "External id": 2482244,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660498.647, "dur": 26.427,
    "args": {
      "External id": 2482245,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660544.409, "dur": 6.568,
    "args": {
      "External id": 2482246,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660545.944, "dur": 4.256,
    "args": {
      "External id": 2482247,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660548.197, "dur": 1.663,
    "args": {
      "External id": 2482248,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660553.996, "dur": 48.654,
    "args": {
      "External id": 2482249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660557.187, "dur": 44.875,
    "args": {
      "External id": 2482250,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660606.540, "dur": 16.881,
    "args": {
      "External id": 2482251,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660628.461, "dur": 26.992,
    "args": {
      "External id": 2482252,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660630.848, "dur": 24.200,
    "args": {
      "External id": 2482253,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660636.415, "dur": 2.500,
    "args": {
      "External id": 2482254,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660660.627, "dur": 29.268,
    "args": {
      "External id": 2482255,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660662.394, "dur": 27.102,
    "args": {
      "External id": 2482256,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660667.170, "dur": 4.364,
    "args": {
      "External id": 2482257,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660674.863, "dur": 14.096,
    "args": {
      "External id": 2482258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660702.810, "dur": 4.994,
    "args": {
      "External id": 2482259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660704.662, "dur": 2.822,
    "args": {
      "External id": 2482260,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660709.063, "dur": 1.605,
    "args": {
      "External id": 2482261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660710.001, "dur": 0.593,
    "args": {
      "External id": 2482262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660752.031, "dur": 22.487,
    "args": {
      "External id": 2482263,"Sequence number": 24740640, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660776.738, "dur": 16.326,
    "args": {
      "External id": 2482264,"Sequence number": 24740641, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9967
    }
  },
  {
    "ph": "s", "id": 9, "pid": 1336760, "tid": 1336760, "ts": 1514656660776.738,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660799.416, "dur": 5.861,
    "args": {
      "External id": 2482265,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660802.863, "dur": 1.047,
    "args": {
      "External id": 2482266,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660812.065, "dur": 8.845,
    "args": {
      "External id": 2482267,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "3"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660818.795, "dur": 0.740,
    "args": {
      "External id": 2482268,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660822.393, "dur": 3.154,
    "args": {
      "External id": 2482269,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660824.027, "dur": 0.969,
    "args": {
      "External id": 2482270,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660831.560, "dur": 6.295,
    "args": {
      "External id": 2482271,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9974
    }
  },
  {
    "ph": "s", "id": 8, "pid": 1336760, "tid": 1336760, "ts": 1514656660831.560,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660835.465, "dur": 1.148,
    "args": {
      "External id": 2482272,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660838.978, "dur": 4.656,
    "args": {
      "External id": 2482273,"Sequence number": 24740643, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9976
    }
  },
  {
    "ph": "s", "id": 7, "pid": 1336760, "tid": 1336760, "ts": 1514656660838.978,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660842.005, "dur": 0.878,
    "args": {
      "External id": 2482274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660844.508, "dur": 7.145,
    "args": {
      "External id": 2482275,"Sequence number": 24740644, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9978
    }
  },
  {
    "ph": "s", "id": 6, "pid": 1336760, "tid": 1336760, "ts": 1514656660844.508,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660850.169, "dur": 0.654,
    "args": {
      "External id": 2482276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660852.654, "dur": 6.541,
    "args": {
      "External id": 2482277,"Sequence number": 24740645, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9980
    }
  },
  {
    "ph": "s", "id": 5, "pid": 1336760, "tid": 1336760, "ts": 1514656660852.654,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660855.777, "dur": 2.668,
    "args": {
      "External id": 2482278,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660863.158, "dur": 29.402,
    "args": {
      "External id": 2482279,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660865.064, "dur": 27.291,
    "args": {
      "External id": 2482280,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660867.620, "dur": 5.902,
    "args": {
      "External id": 2482281,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660869.840, "dur": 3.093,
    "args": {
      "External id": 2482282,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660874.202, "dur": 17.658,
    "args": {
      "External id": 2482283,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660917.297, "dur": 5.261,
    "args": {
      "External id": 2482284,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9987
    }
  },
  {
    "ph": "s", "id": 4, "pid": 1336760, "tid": 1336760, "ts": 1514656660917.297,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660924.967, "dur": 1.611,
    "args": {
      "External id": 2482285,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660958.787, "dur": 44010.207,
    "args": {
      "External id": 2482286,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9989
    }
  },
  {
    "ph": "s", "id": 3, "pid": 1336760, "tid": 1336760, "ts": 1514656660958.787,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660972.151, "dur": 65.719,
    "args": {
      "External id": 2482287,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660972.762, "dur": 64.840,
    "args": {
      "External id": 2482288,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660974.644, "dur": 6.691,
    "args": {
      "External id": 2482289,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660975.735, "dur": 5.163,
    "args": {
      "External id": 2482290,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656660982.110, "dur": 54.699,
    "args": {
      "External id": 2482291,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661055.803, "dur": 30.901,
    "args": {
      "External id": 2482292,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661056.761, "dur": 10.126,
    "args": {
      "External id": 2482293,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661061.424, "dur": 5.166,
    "args": {
      "External id": 2482294,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661068.198, "dur": 18.287,
    "args": {
      "External id": 2482295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661070.137, "dur": 15.955,
    "args": {
      "External id": 2482296,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661090.189, "dur": 20.260,
    "args": {
      "External id": 2482297,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661091.036, "dur": 4.856,
    "args": {
      "External id": 2482298,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661092.516, "dur": 3.147,
    "args": {
      "External id": 2482299,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661096.647, "dur": 13.612,
    "args": {
      "External id": 2482300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661097.370, "dur": 12.518,
    "args": {
      "External id": 2482301,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 10004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661116.332, "dur": 20.566,
    "args": {
      "External id": 2482302,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661119.822, "dur": 3.595,
    "args": {
      "External id": 2482303,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661124.045, "dur": 12.596,
    "args": {
      "External id": 2482304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 10007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661124.588, "dur": 11.759,
    "args": {
      "External id": 2482305,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661141.535, "dur": 22.670,
    "args": {
      "External id": 2482306,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661166.845, "dur": 52.419,
    "args": {
      "External id": 2482307,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661168.883, "dur": 49.997,
    "args": {
      "External id": 2482308,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661176.229, "dur": 0.807,
    "args": {
      "External id": 2482309,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661178.146, "dur": 23.678,
    "args": {
      "External id": 2482310,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661179.907, "dur": 21.594,
    "args": {
      "External id": 2482311,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 10014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661182.383, "dur": 3.275,
    "args": {
      "External id": 2482312,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661186.634, "dur": 14.577,
    "args": {
      "External id": 2482313,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 10016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661225.716, "dur": 37931.613,
    "args": {
      "External id": 2482314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514656661227.124, "dur": 37929.542,
    "args": {
      "External id": 2482315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699166.497, "dur": 6.364,
    "args": {
      "External id": 2482316,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699170.088, "dur": 1.045,
    "args": {
      "External id": 2482317,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699177.721, "dur": 106.203,
    "args": {
      "External id": 2482318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699179.628, "dur": 8.000,
    "args": {
      "External id": 2482319,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699181.994, "dur": 4.746,
    "args": {
      "External id": 2482320,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699185.613, "dur": 0.867,
    "args": {
      "External id": 2482321,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699188.813, "dur": 94.430,
    "args": {
      "External id": 2482322,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699190.473, "dur": 91.901,
    "args": {
      "External id": 2482323,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699287.683, "dur": 4.465,
    "args": {
      "External id": 2482324,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699289.944, "dur": 1.053,
    "args": {
      "External id": 2482325,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699299.156, "dur": 2.929,
    "args": {
      "External id": 2482326,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699313.787, "dur": 6.363,
    "args": {
      "External id": 2482327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699315.748, "dur": 4.136,
    "args": {
      "External id": 2482328,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699449.451, "dur": 232.080,
    "args": {
      "External id": 2482329,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699454.409, "dur": 2.512,
    "args": {
      "External id": 2482330,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699459.086, "dur": 221.912,
    "args": {
      "External id": 2482331,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699460.578, "dur": 0.485,
    "args": {
      "External id": 2482332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699462.383, "dur": 26.839,
    "args": {
      "External id": 2482333,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699493.222, "dur": 5.636,
    "args": {
      "External id": 2482334,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699497.436, "dur": 1.132,
    "args": {
      "External id": 2482335,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699499.707, "dur": 25.261,
    "args": {
      "External id": 2482336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699501.137, "dur": 2.074,
    "args": {
      "External id": 2482337,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699504.811, "dur": 19.724,
    "args": {
      "External id": 2482338,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699508.263, "dur": 3.200,
    "args": {
      "External id": 2482339,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699526.676, "dur": 39.043,
    "args": {
      "External id": 2482340,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699568.303, "dur": 19.516,
    "args": {
      "External id": 2482341,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699591.094, "dur": 16.445,
    "args": {
      "External id": 2482342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699608.971, "dur": 15.328,
    "args": {
      "External id": 2482343,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699628.409, "dur": 23.198,
    "args": {
      "External id": 2482344,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699630.672, "dur": 2.149,
    "args": {
      "External id": 2482345,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699635.423, "dur": 0.947,
    "args": {
      "External id": 2482346,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699653.246, "dur": 13.018,
    "args": {
      "External id": 2482347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699667.777, "dur": 11.976,
    "args": {
      "External id": 2482348,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699689.493, "dur": 2.388,
    "args": {
      "External id": 2482349,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699701.032, "dur": 4.409,
    "args": {
      "External id": 2482350,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699703.718, "dur": 0.922,
    "args": {
      "External id": 2482351,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699783.936, "dur": 70.008,
    "args": {
      "External id": 2482352,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699859.620, "dur": 5.526,
    "args": {
      "External id": 2482353,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699862.167, "dur": 0.713,
    "args": {
      "External id": 2482354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699866.846, "dur": 28.154,
    "args": {
      "External id": 2482355,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699900.396, "dur": 8.754,
    "args": {
      "External id": 2482356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699902.161, "dur": 6.120,
    "args": {
      "External id": 2482357,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699906.404, "dur": 1.611,
    "args": {
      "External id": 2482358,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699912.575, "dur": 45.230,
    "args": {
      "External id": 2482359,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699913.952, "dur": 43.241,
    "args": {
      "External id": 2482360,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656699962.021, "dur": 16.617,
    "args": {
      "External id": 2482361,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700025.505, "dur": 5.374,
    "args": {
      "External id": 2482362,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700028.306, "dur": 1.118,
    "args": {
      "External id": 2482363,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700038.573, "dur": 58.056,
    "args": {
      "External id": 2482364,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700039.776, "dur": 4.284,
    "args": {
      "External id": 2482365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700040.770, "dur": 2.655,
    "args": {
      "External id": 2482366,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700042.421, "dur": 0.859,
    "args": {
      "External id": 2482367,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700044.693, "dur": 51.513,
    "args": {
      "External id": 2482368,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700045.964, "dur": 49.639,
    "args": {
      "External id": 2482369,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700102.857, "dur": 4.281,
    "args": {
      "External id": 2482370,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700105.166, "dur": 0.825,
    "args": {
      "External id": 2482371,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700114.343, "dur": 1.819,
    "args": {
      "External id": 2482372,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700124.462, "dur": 8.737,
    "args": {
      "External id": 2482373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700126.882, "dur": 6.032,
    "args": {
      "External id": 2482374,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700236.255, "dur": 198.398,
    "args": {
      "External id": 2482375,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700239.004, "dur": 2.413,
    "args": {
      "External id": 2482376,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700242.775, "dur": 191.345,
    "args": {
      "External id": 2482377,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700244.398, "dur": 0.358,
    "args": {
      "External id": 2482378,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700246.115, "dur": 26.708,
    "args": {
      "External id": 2482379,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700274.466, "dur": 3.684,
    "args": {
      "External id": 2482380,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700276.801, "dur": 0.923,
    "args": {
      "External id": 2482381,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700279.067, "dur": 27.309,
    "args": {
      "External id": 2482382,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700280.430, "dur": 1.944,
    "args": {
      "External id": 2482383,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700283.384, "dur": 22.678,
    "args": {
      "External id": 2482384,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700290.278, "dur": 2.815,
    "args": {
      "External id": 2482385,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700307.758, "dur": 23.412,
    "args": {
      "External id": 2482386,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700332.964, "dur": 14.319,
    "args": {
      "External id": 2482387,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700350.247, "dur": 15.151,
    "args": {
      "External id": 2482388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700366.554, "dur": 13.704,
    "args": {
      "External id": 2482389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700382.219, "dur": 20.929,
    "args": {
      "External id": 2482390,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700384.196, "dur": 1.210,
    "args": {
      "External id": 2482391,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700387.528, "dur": 1.154,
    "args": {
      "External id": 2482392,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700406.629, "dur": 13.109,
    "args": {
      "External id": 2482393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700420.804, "dur": 12.429,
    "args": {
      "External id": 2482394,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700440.593, "dur": 1.853,
    "args": {
      "External id": 2482395,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700450.459, "dur": 3.824,
    "args": {
      "External id": 2482396,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700452.784, "dur": 0.568,
    "args": {
      "External id": 2482397,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700521.100, "dur": 73.206,
    "args": {
      "External id": 2482398,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700600.954, "dur": 6.150,
    "args": {
      "External id": 2482399,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700604.178, "dur": 1.403,
    "args": {
      "External id": 2482400,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700610.366, "dur": 27.810,
    "args": {
      "External id": 2482401,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700644.012, "dur": 5.523,
    "args": {
      "External id": 2482402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700645.582, "dur": 3.400,
    "args": {
      "External id": 2482403,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700647.628, "dur": 1.174,
    "args": {
      "External id": 2482404,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700652.531, "dur": 43.020,
    "args": {
      "External id": 2482405,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700653.665, "dur": 41.291,
    "args": {
      "External id": 2482406,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700699.210, "dur": 16.501,
    "args": {
      "External id": 2482407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700723.158, "dur": 3.412,
    "args": {
      "External id": 2482408,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700725.239, "dur": 0.511,
    "args": {
      "External id": 2482409,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700730.720, "dur": 52.791,
    "args": {
      "External id": 2482410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700731.754, "dur": 6.122,
    "args": {
      "External id": 2482411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700732.616, "dur": 4.669,
    "args": {
      "External id": 2482412,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700734.331, "dur": 2.746,
    "args": {
      "External id": 2482413,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700740.835, "dur": 42.212,
    "args": {
      "External id": 2482414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700741.683, "dur": 40.699,
    "args": {
      "External id": 2482415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700787.284, "dur": 3.104,
    "args": {
      "External id": 2482416,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700788.829, "dur": 0.720,
    "args": {
      "External id": 2482417,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700796.349, "dur": 1.501,
    "args": {
      "External id": 2482418,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700805.267, "dur": 6.837,
    "args": {
      "External id": 2482419,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700807.426, "dur": 4.401,
    "args": {
      "External id": 2482420,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700899.539, "dur": 234.370,
    "args": {
      "External id": 2482421,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700902.146, "dur": 2.213,
    "args": {
      "External id": 2482422,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700905.598, "dur": 227.874,
    "args": {
      "External id": 2482423,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700907.119, "dur": 0.403,
    "args": {
      "External id": 2482424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700911.229, "dur": 24.314,
    "args": {
      "External id": 2482425,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700937.015, "dur": 3.751,
    "args": {
      "External id": 2482426,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700939.147, "dur": 1.193,
    "args": {
      "External id": 2482427,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700941.750, "dur": 26.803,
    "args": {
      "External id": 2482428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700942.838, "dur": 3.741,
    "args": {
      "External id": 2482429,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700947.643, "dur": 20.425,
    "args": {
      "External id": 2482430,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700952.555, "dur": 2.847,
    "args": {
      "External id": 2482431,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656700969.841, "dur": 55.848,
    "args": {
      "External id": 2482432,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701028.243, "dur": 16.320,
    "args": {
      "External id": 2482433,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701047.996, "dur": 15.373,
    "args": {
      "External id": 2482434,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701064.775, "dur": 14.248,
    "args": {
      "External id": 2482435,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701081.095, "dur": 22.814,
    "args": {
      "External id": 2482436,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701082.925, "dur": 2.001,
    "args": {
      "External id": 2482437,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701087.066, "dur": 0.941,
    "args": {
      "External id": 2482438,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701107.567, "dur": 12.523,
    "args": {
      "External id": 2482439,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701121.002, "dur": 11.298,
    "args": {
      "External id": 2482440,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701141.929, "dur": 2.020,
    "args": {
      "External id": 2482441,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701153.857, "dur": 4.036,
    "args": {
      "External id": 2482442,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701156.504, "dur": 0.604,
    "args": {
      "External id": 2482443,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701230.517, "dur": 83.111,
    "args": {
      "External id": 2482444,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701318.756, "dur": 6.725,
    "args": {
      "External id": 2482445,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701322.945, "dur": 1.533,
    "args": {
      "External id": 2482446,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701327.010, "dur": 25.477,
    "args": {
      "External id": 2482447,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701357.114, "dur": 7.633,
    "args": {
      "External id": 2482448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701358.766, "dur": 5.261,
    "args": {
      "External id": 2482449,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701360.589, "dur": 3.226,
    "args": {
      "External id": 2482450,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701367.603, "dur": 57.061,
    "args": {
      "External id": 2482451,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701368.646, "dur": 55.397,
    "args": {
      "External id": 2482452,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701430.433, "dur": 16.530,
    "args": {
      "External id": 2482453,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701452.329, "dur": 3.519,
    "args": {
      "External id": 2482454,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701454.386, "dur": 0.636,
    "args": {
      "External id": 2482455,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701459.862, "dur": 50.749,
    "args": {
      "External id": 2482456,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701460.614, "dur": 5.789,
    "args": {
      "External id": 2482457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701461.369, "dur": 4.567,
    "args": {
      "External id": 2482458,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701464.762, "dur": 0.880,
    "args": {
      "External id": 2482459,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701467.158, "dur": 42.985,
    "args": {
      "External id": 2482460,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701467.985, "dur": 41.610,
    "args": {
      "External id": 2482461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701514.445, "dur": 35.725,
    "args": {
      "External id": 2482462,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701547.248, "dur": 1.085,
    "args": {
      "External id": 2482463,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701558.335, "dur": 1.862,
    "args": {
      "External id": 2482464,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701569.855, "dur": 9.296,
    "args": {
      "External id": 2482465,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701571.867, "dur": 7.016,
    "args": {
      "External id": 2482466,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701666.948, "dur": 196.851,
    "args": {
      "External id": 2482467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701669.111, "dur": 2.407,
    "args": {
      "External id": 2482468,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701672.948, "dur": 190.354,
    "args": {
      "External id": 2482469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701674.408, "dur": 0.318,
    "args": {
      "External id": 2482470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701675.860, "dur": 31.344,
    "args": {
      "External id": 2482471,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701710.579, "dur": 3.827,
    "args": {
      "External id": 2482472,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701713.080, "dur": 1.029,
    "args": {
      "External id": 2482473,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701715.514, "dur": 23.660,
    "args": {
      "External id": 2482474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701716.525, "dur": 1.536,
    "args": {
      "External id": 2482475,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701719.146, "dur": 19.688,
    "args": {
      "External id": 2482476,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701723.778, "dur": 2.605,
    "args": {
      "External id": 2482477,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701740.552, "dur": 22.636,
    "args": {
      "External id": 2482478,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701764.545, "dur": 13.867,
    "args": {
      "External id": 2482479,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701781.373, "dur": 15.351,
    "args": {
      "External id": 2482480,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701798.285, "dur": 12.919,
    "args": {
      "External id": 2482481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701815.179, "dur": 20.337,
    "args": {
      "External id": 2482482,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701817.324, "dur": 1.405,
    "args": {
      "External id": 2482483,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701820.932, "dur": 0.694,
    "args": {
      "External id": 2482484,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701836.767, "dur": 12.696,
    "args": {
      "External id": 2482485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701850.431, "dur": 11.981,
    "args": {
      "External id": 2482486,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701870.131, "dur": 1.748,
    "args": {
      "External id": 2482487,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701879.657, "dur": 3.392,
    "args": {
      "External id": 2482488,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701882.005, "dur": 0.246,
    "args": {
      "External id": 2482489,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656701951.249, "dur": 96.606,
    "args": {
      "External id": 2482490,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702054.806, "dur": 5.770,
    "args": {
      "External id": 2482491,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702057.927, "dur": 1.141,
    "args": {
      "External id": 2482492,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702062.036, "dur": 26.727,
    "args": {
      "External id": 2482493,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702094.526, "dur": 7.904,
    "args": {
      "External id": 2482494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702095.951, "dur": 5.842,
    "args": {
      "External id": 2482495,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702100.191, "dur": 1.359,
    "args": {
      "External id": 2482496,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702105.387, "dur": 45.178,
    "args": {
      "External id": 2482497,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702106.485, "dur": 43.487,
    "args": {
      "External id": 2482498,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702154.345, "dur": 16.683,
    "args": {
      "External id": 2482499,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702176.808, "dur": 4.221,
    "args": {
      "External id": 2482500,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702179.326, "dur": 0.836,
    "args": {
      "External id": 2482501,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702185.158, "dur": 50.064,
    "args": {
      "External id": 2482502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702188.300, "dur": 3.719,
    "args": {
      "External id": 2482503,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702189.355, "dur": 2.144,
    "args": {
      "External id": 2482504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702190.779, "dur": 0.610,
    "args": {
      "External id": 2482505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702192.938, "dur": 41.828,
    "args": {
      "External id": 2482506,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702193.789, "dur": 40.483,
    "args": {
      "External id": 2482507,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702239.551, "dur": 6.900,
    "args": {
      "External id": 2482508,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702241.751, "dur": 3.623,
    "args": {
      "External id": 2482509,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702251.973, "dur": 1.764,
    "args": {
      "External id": 2482510,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702263.556, "dur": 6.557,
    "args": {
      "External id": 2482511,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702265.737, "dur": 4.119,
    "args": {
      "External id": 2482512,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702359.854, "dur": 218.793,
    "args": {
      "External id": 2482513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702362.235, "dur": 2.500,
    "args": {
      "External id": 2482514,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702366.734, "dur": 211.392,
    "args": {
      "External id": 2482515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702367.970, "dur": 0.298,
    "args": {
      "External id": 2482516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702371.153, "dur": 23.901,
    "args": {
      "External id": 2482517,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702402.516, "dur": 5.326,
    "args": {
      "External id": 2482518,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702406.338, "dur": 0.995,
    "args": {
      "External id": 2482519,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702408.747, "dur": 25.121,
    "args": {
      "External id": 2482520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702409.956, "dur": 3.607,
    "args": {
      "External id": 2482521,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702414.739, "dur": 18.875,
    "args": {
      "External id": 2482522,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702417.065, "dur": 2.686,
    "args": {
      "External id": 2482523,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702435.242, "dur": 21.381,
    "args": {
      "External id": 2482524,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702458.147, "dur": 14.735,
    "args": {
      "External id": 2482525,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702478.337, "dur": 14.858,
    "args": {
      "External id": 2482526,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702494.584, "dur": 13.633,
    "args": {
      "External id": 2482527,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702509.831, "dur": 37.290,
    "args": {
      "External id": 2482528,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702511.784, "dur": 1.235,
    "args": {
      "External id": 2482529,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702514.925, "dur": 0.732,
    "args": {
      "External id": 2482530,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702549.587, "dur": 14.906,
    "args": {
      "External id": 2482531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702565.639, "dur": 11.435,
    "args": {
      "External id": 2482532,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702588.367, "dur": 2.430,
    "args": {
      "External id": 2482533,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702598.786, "dur": 3.366,
    "args": {
      "External id": 2482534,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702600.917, "dur": 0.407,
    "args": {
      "External id": 2482535,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702669.316, "dur": 55.121,
    "args": {
      "External id": 2482536,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702729.394, "dur": 4.631,
    "args": {
      "External id": 2482537,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702731.990, "dur": 0.936,
    "args": {
      "External id": 2482538,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702735.523, "dur": 23.620,
    "args": {
      "External id": 2482539,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702763.282, "dur": 10.124,
    "args": {
      "External id": 2482540,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702767.417, "dur": 5.322,
    "args": {
      "External id": 2482541,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702769.170, "dur": 3.323,
    "args": {
      "External id": 2482542,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702775.978, "dur": 40.389,
    "args": {
      "External id": 2482543,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702776.975, "dur": 38.774,
    "args": {
      "External id": 2482544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702819.831, "dur": 15.314,
    "args": {
      "External id": 2482545,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702840.762, "dur": 6.141,
    "args": {
      "External id": 2482546,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702845.125, "dur": 0.940,
    "args": {
      "External id": 2482547,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702850.265, "dur": 47.835,
    "args": {
      "External id": 2482548,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702851.147, "dur": 3.694,
    "args": {
      "External id": 2482549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702851.945, "dur": 2.420,
    "args": {
      "External id": 2482550,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702853.152, "dur": 0.948,
    "args": {
      "External id": 2482551,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702855.354, "dur": 42.364,
    "args": {
      "External id": 2482552,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702856.213, "dur": 40.851,
    "args": {
      "External id": 2482553,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702903.765, "dur": 3.766,
    "args": {
      "External id": 2482554,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702905.666, "dur": 0.933,
    "args": {
      "External id": 2482555,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702912.400, "dur": 1.451,
    "args": {
      "External id": 2482556,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702921.551, "dur": 6.431,
    "args": {
      "External id": 2482557,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656702923.262, "dur": 4.458,
    "args": {
      "External id": 2482558,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703048.549, "dur": 193.445,
    "args": {
      "External id": 2482559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703051.261, "dur": 3.338,
    "args": {
      "External id": 2482560,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703058.943, "dur": 182.545,
    "args": {
      "External id": 2482561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703060.272, "dur": 0.456,
    "args": {
      "External id": 2482562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703061.870, "dur": 23.979,
    "args": {
      "External id": 2482563,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703087.288, "dur": 5.488,
    "args": {
      "External id": 2482564,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703089.359, "dur": 2.924,
    "args": {
      "External id": 2482565,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703093.649, "dur": 24.452,
    "args": {
      "External id": 2482566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703094.948, "dur": 1.472,
    "args": {
      "External id": 2482567,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703097.636, "dur": 20.057,
    "args": {
      "External id": 2482568,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703102.339, "dur": 2.658,
    "args": {
      "External id": 2482569,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703119.654, "dur": 22.113,
    "args": {
      "External id": 2482570,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703143.284, "dur": 13.566,
    "args": {
      "External id": 2482571,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703159.822, "dur": 14.583,
    "args": {
      "External id": 2482572,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703175.747, "dur": 13.154,
    "args": {
      "External id": 2482573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703190.462, "dur": 21.622,
    "args": {
      "External id": 2482574,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703192.867, "dur": 1.797,
    "args": {
      "External id": 2482575,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703196.531, "dur": 1.633,
    "args": {
      "External id": 2482576,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703215.434, "dur": 12.100,
    "args": {
      "External id": 2482577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703228.660, "dur": 11.822,
    "args": {
      "External id": 2482578,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703248.361, "dur": 2.177,
    "args": {
      "External id": 2482579,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703259.811, "dur": 3.667,
    "args": {
      "External id": 2482580,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703261.856, "dur": 0.813,
    "args": {
      "External id": 2482581,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703324.939, "dur": 55.183,
    "args": {
      "External id": 2482582,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703384.826, "dur": 9.155,
    "args": {
      "External id": 2482583,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703389.268, "dur": 3.715,
    "args": {
      "External id": 2482584,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703395.576, "dur": 25.218,
    "args": {
      "External id": 2482585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703425.938, "dur": 5.366,
    "args": {
      "External id": 2482586,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703427.689, "dur": 3.085,
    "args": {
      "External id": 2482587,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703429.453, "dur": 1.097,
    "args": {
      "External id": 2482588,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703433.616, "dur": 39.631,
    "args": {
      "External id": 2482589,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703434.663, "dur": 37.971,
    "args": {
      "External id": 2482590,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703478.494, "dur": 15.498,
    "args": {
      "External id": 2482591,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703499.237, "dur": 3.850,
    "args": {
      "External id": 2482592,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703501.519, "dur": 0.759,
    "args": {
      "External id": 2482593,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703506.628, "dur": 66.904,
    "args": {
      "External id": 2482594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703507.419, "dur": 5.989,
    "args": {
      "External id": 2482595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703508.402, "dur": 4.274,
    "args": {
      "External id": 2482596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703512.071, "dur": 0.504,
    "args": {
      "External id": 2482597,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703513.975, "dur": 59.077,
    "args": {
      "External id": 2482598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703514.700, "dur": 57.384,
    "args": {
      "External id": 2482599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703579.189, "dur": 4.197,
    "args": {
      "External id": 2482600,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703581.705, "dur": 0.616,
    "args": {
      "External id": 2482601,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703589.142, "dur": 1.713,
    "args": {
      "External id": 2482602,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703598.191, "dur": 8.554,
    "args": {
      "External id": 2482603,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703601.844, "dur": 4.667,
    "args": {
      "External id": 2482604,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703686.652, "dur": 188.762,
    "args": {
      "External id": 2482605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703688.635, "dur": 4.286,
    "args": {
      "External id": 2482606,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703694.564, "dur": 180.198,
    "args": {
      "External id": 2482607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703696.081, "dur": 0.285,
    "args": {
      "External id": 2482608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703697.412, "dur": 22.990,
    "args": {
      "External id": 2482609,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703721.939, "dur": 5.713,
    "args": {
      "External id": 2482610,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703726.641, "dur": 0.789,
    "args": {
      "External id": 2482611,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703728.449, "dur": 24.720,
    "args": {
      "External id": 2482612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703731.772, "dur": 1.586,
    "args": {
      "External id": 2482613,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703734.759, "dur": 18.132,
    "args": {
      "External id": 2482614,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703736.994, "dur": 2.396,
    "args": {
      "External id": 2482615,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703754.368, "dur": 20.702,
    "args": {
      "External id": 2482616,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703776.483, "dur": 14.062,
    "args": {
      "External id": 2482617,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703793.215, "dur": 14.188,
    "args": {
      "External id": 2482618,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703808.393, "dur": 13.241,
    "args": {
      "External id": 2482619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703823.022, "dur": 23.166,
    "args": {
      "External id": 2482620,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703826.443, "dur": 1.150,
    "args": {
      "External id": 2482621,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703831.334, "dur": 0.898,
    "args": {
      "External id": 2482622,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703847.431, "dur": 13.732,
    "args": {
      "External id": 2482623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703862.407, "dur": 11.447,
    "args": {
      "External id": 2482624,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703881.390, "dur": 1.458,
    "args": {
      "External id": 2482625,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703891.042, "dur": 3.284,
    "args": {
      "External id": 2482626,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703893.151, "dur": 0.359,
    "args": {
      "External id": 2482627,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656703952.024, "dur": 92.097,
    "args": {
      "External id": 2482628,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704051.286, "dur": 8.050,
    "args": {
      "External id": 2482629,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704056.677, "dur": 1.227,
    "args": {
      "External id": 2482630,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704060.706, "dur": 24.929,
    "args": {
      "External id": 2482631,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704090.900, "dur": 5.560,
    "args": {
      "External id": 2482632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704092.534, "dur": 3.314,
    "args": {
      "External id": 2482633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704094.536, "dur": 1.119,
    "args": {
      "External id": 2482634,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704099.427, "dur": 46.664,
    "args": {
      "External id": 2482635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704102.762, "dur": 42.501,
    "args": {
      "External id": 2482636,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704149.761, "dur": 15.700,
    "args": {
      "External id": 2482637,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704171.060, "dur": 3.876,
    "args": {
      "External id": 2482638,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704173.455, "dur": 0.608,
    "args": {
      "External id": 2482639,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704178.661, "dur": 48.883,
    "args": {
      "External id": 2482640,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704179.666, "dur": 5.420,
    "args": {
      "External id": 2482641,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704180.358, "dur": 4.178,
    "args": {
      "External id": 2482642,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704183.790, "dur": 0.618,
    "args": {
      "External id": 2482643,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704185.728, "dur": 41.351,
    "args": {
      "External id": 2482644,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704186.175, "dur": 40.329,
    "args": {
      "External id": 2482645,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704230.826, "dur": 3.912,
    "args": {
      "External id": 2482646,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704232.991, "dur": 0.834,
    "args": {
      "External id": 2482647,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704240.595, "dur": 1.417,
    "args": {
      "External id": 2482648,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704251.041, "dur": 5.866,
    "args": {
      "External id": 2482649,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704252.643, "dur": 4.040,
    "args": {
      "External id": 2482650,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704338.782, "dur": 209.812,
    "args": {
      "External id": 2482651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704342.845, "dur": 2.339,
    "args": {
      "External id": 2482652,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704346.887, "dur": 201.158,
    "args": {
      "External id": 2482653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704347.864, "dur": 0.468,
    "args": {
      "External id": 2482654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704351.633, "dur": 21.547,
    "args": {
      "External id": 2482655,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704374.693, "dur": 5.227,
    "args": {
      "External id": 2482656,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704378.891, "dur": 0.819,
    "args": {
      "External id": 2482657,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704380.784, "dur": 21.712,
    "args": {
      "External id": 2482658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704382.226, "dur": 1.638,
    "args": {
      "External id": 2482659,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704384.987, "dur": 17.218,
    "args": {
      "External id": 2482660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704387.432, "dur": 2.347,
    "args": {
      "External id": 2482661,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704403.979, "dur": 21.729,
    "args": {
      "External id": 2482662,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704427.090, "dur": 16.981,
    "args": {
      "External id": 2482663,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704446.929, "dur": 15.601,
    "args": {
      "External id": 2482664,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704465.728, "dur": 13.025,
    "args": {
      "External id": 2482665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704480.529, "dur": 20.931,
    "args": {
      "External id": 2482666,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704482.170, "dur": 1.397,
    "args": {
      "External id": 2482667,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704485.907, "dur": 0.601,
    "args": {
      "External id": 2482668,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704502.843, "dur": 12.873,
    "args": {
      "External id": 2482669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704516.872, "dur": 29.233,
    "args": {
      "External id": 2482670,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704556.785, "dur": 2.338,
    "args": {
      "External id": 2482671,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704569.679, "dur": 3.719,
    "args": {
      "External id": 2482672,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704572.070, "dur": 0.575,
    "args": {
      "External id": 2482673,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704643.989, "dur": 54.446,
    "args": {
      "External id": 2482674,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704703.386, "dur": 4.072,
    "args": {
      "External id": 2482675,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704705.616, "dur": 0.861,
    "args": {
      "External id": 2482676,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704708.594, "dur": 25.653,
    "args": {
      "External id": 2482677,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704738.490, "dur": 7.952,
    "args": {
      "External id": 2482678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704740.134, "dur": 5.745,
    "args": {
      "External id": 2482679,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704744.481, "dur": 1.160,
    "args": {
      "External id": 2482680,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704749.075, "dur": 40.024,
    "args": {
      "External id": 2482681,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704749.950, "dur": 38.623,
    "args": {
      "External id": 2482682,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704792.715, "dur": 17.937,
    "args": {
      "External id": 2482683,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704815.540, "dur": 27.306,
    "args": {
      "External id": 2482684,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704817.701, "dur": 24.712,
    "args": {
      "External id": 2482685,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704822.524, "dur": 2.533,
    "args": {
      "External id": 2482686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704847.630, "dur": 77.605,
    "args": {
      "External id": 2482687,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704849.253, "dur": 75.680,
    "args": {
      "External id": 2482688,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 10391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704902.577, "dur": 4.726,
    "args": {
      "External id": 2482689,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704908.831, "dur": 15.474,
    "args": {
      "External id": 2482690,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704937.905, "dur": 4.788,
    "args": {
      "External id": 2482691,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704939.493, "dur": 2.861,
    "args": {
      "External id": 2482692,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704943.718, "dur": 0.918,
    "args": {
      "External id": 2482693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336760, "tid": 1336760,
    "ts": 1514656704944.103, "dur": 0.455,
    "args": {
      "External id": 2482694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705022.520, "dur": 30.646,
    "args": {
      "External id": 2482695,"Sequence number": 24740648, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705055.631, "dur": 14.394,
    "args": {
      "External id": 2482696,"Sequence number": 24740649, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10399
    }
  },
  {
    "ph": "s", "id": 2, "pid": 1336760, "tid": 1336760, "ts": 1514656705055.631,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705177.829, "dur": 40.249,
    "args": {
      "External id": 2482697,"Record function id": 0, "Ev Idx": 10400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705320.053, "dur": 35.418,
    "args": {
      "External id": 2482698,"Sequence number": 24740650, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10401
    }
  },
  {
    "ph": "s", "id": 1, "pid": 1336760, "tid": 1336760, "ts": 1514656705320.053,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ones_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705415.571, "dur": 26.782,
    "args": {
      "External id": 2482699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705416.842, "dur": 7.959,
    "args": {
      "External id": 2482700,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705420.568, "dur": 3.601,
    "args": {
      "External id": 2482701,"Record function id": 0, "Concrete Inputs": ["[1]", "[1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514656705426.384, "dur": 15.699,
    "args": {
      "External id": 2482702,"Record function id": 0, "Concrete Inputs": ["", "1."], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336760, "tid": 1336760,
    "ts": 1514658027259.174, "dur": 87.772,
    "args": {
      "External id": 2482703,"Sequence number": 24740651, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336760, "tid": 1336760,
    "ts": 1514658027359.917, "dur": 26.006,
    "args": {
      "External id": 2482704,"Sequence number": 24740652, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514658027395.936, "dur": 25.193,
    "args": {
      "External id": 2482705,"Sequence number": 24740653, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514658028045.475, "dur": 32.725,
    "args": {
      "External id": 2482706,"Sequence number": 24740654, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514658028732.621, "dur": 35.436,
    "args": {
      "External id": 2482707,"Sequence number": 24740655, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336760, "tid": 1336760,
    "ts": 1514658030650.455, "dur": 3394.281,
    "args": {
      "External id": 2482708,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336760, "tid": 1336760,
    "ts": 1514658031326.771, "dur": 1122.657,
    "args": {
      "External id": 2482709,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336760, "tid": 1336760,
    "ts": 1514658031347.176, "dur": 70.563,
    "args": {
      "External id": 2482710,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514658031350.711, "dur": 14.057,
    "args": {
      "External id": 2482711,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658031370.678, "dur": 46.759,
    "args": {
      "External id": 2482712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[36375]], "Ev Idx": 10415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658031373.040, "dur": 43.819,
    "args": {
      "External id": 2482713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[36375], []], "Ev Idx": 10416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034082.043, "dur": 2.925,
    "args": {
      "External id": 2482714,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034087.294, "dur": 0.789,
    "args": {
      "External id": 2482715,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034089.245, "dur": 0.669,
    "args": {
      "External id": 2482716,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034091.382, "dur": 0.599,
    "args": {
      "External id": 2482717,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034093.574, "dur": 0.459,
    "args": {
      "External id": 2482718,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034095.442, "dur": 0.423,
    "args": {
      "External id": 2482719,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034096.777, "dur": 0.610,
    "args": {
      "External id": 2482720,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034100.563, "dur": 0.220,
    "args": {
      "External id": 2482721,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034101.565, "dur": 0.229,
    "args": {
      "External id": 2482722,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034102.961, "dur": 0.218,
    "args": {
      "External id": 2482723,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034103.975, "dur": 0.229,
    "args": {
      "External id": 2482724,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034105.193, "dur": 0.415,
    "args": {
      "External id": 2482725,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034106.384, "dur": 0.436,
    "args": {
      "External id": 2482726,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034107.617, "dur": 0.396,
    "args": {
      "External id": 2482727,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034108.767, "dur": 0.426,
    "args": {
      "External id": 2482728,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034112.013, "dur": 0.426,
    "args": {
      "External id": 2482729,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034113.460, "dur": 0.218,
    "args": {
      "External id": 2482730,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034114.543, "dur": 0.398,
    "args": {
      "External id": 2482731,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034115.680, "dur": 0.403,
    "args": {
      "External id": 2482732,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034117.205, "dur": 0.465,
    "args": {
      "External id": 2482733,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034118.587, "dur": 0.392,
    "args": {
      "External id": 2482734,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034119.750, "dur": 0.415,
    "args": {
      "External id": 2482735,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034121.125, "dur": 0.448,
    "args": {
      "External id": 2482736,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034124.393, "dur": 0.216,
    "args": {
      "External id": 2482737,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034125.577, "dur": 0.388,
    "args": {
      "External id": 2482738,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034127.079, "dur": 0.457,
    "args": {
      "External id": 2482739,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034128.685, "dur": 0.227,
    "args": {
      "External id": 2482740,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034129.976, "dur": 0.250,
    "args": {
      "External id": 2482741,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034131.122, "dur": 0.226,
    "args": {
      "External id": 2482742,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034132.303, "dur": 0.238,
    "args": {
      "External id": 2482743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034133.463, "dur": 0.222,
    "args": {
      "External id": 2482744,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034136.823, "dur": 0.222,
    "args": {
      "External id": 2482745,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034137.885, "dur": 0.224,
    "args": {
      "External id": 2482746,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034139.215, "dur": 0.249,
    "args": {
      "External id": 2482747,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034140.207, "dur": 0.220,
    "args": {
      "External id": 2482748,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034141.169, "dur": 0.238,
    "args": {
      "External id": 2482749,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034142.121, "dur": 0.222,
    "args": {
      "External id": 2482750,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034143.093, "dur": 0.221,
    "args": {
      "External id": 2482751,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034144.226, "dur": 0.227,
    "args": {
      "External id": 2482752,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034147.223, "dur": 0.222,
    "args": {
      "External id": 2482753,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034148.360, "dur": 0.244,
    "args": {
      "External id": 2482754,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034149.605, "dur": 0.215,
    "args": {
      "External id": 2482755,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034150.731, "dur": 0.222,
    "args": {
      "External id": 2482756,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034151.834, "dur": 0.209,
    "args": {
      "External id": 2482757,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034152.953, "dur": 0.227,
    "args": {
      "External id": 2482758,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034155.411, "dur": 0.249,
    "args": {
      "External id": 2482759,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034156.498, "dur": 0.214,
    "args": {
      "External id": 2482760,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034159.331, "dur": 0.212,
    "args": {
      "External id": 2482761,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034160.269, "dur": 0.215,
    "args": {
      "External id": 2482762,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034161.287, "dur": 0.219,
    "args": {
      "External id": 2482763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034162.432, "dur": 0.221,
    "args": {
      "External id": 2482764,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034163.491, "dur": 0.208,
    "args": {
      "External id": 2482765,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034164.530, "dur": 0.221,
    "args": {
      "External id": 2482766,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034165.542, "dur": 0.211,
    "args": {
      "External id": 2482767,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034166.478, "dur": 0.216,
    "args": {
      "External id": 2482768,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034169.417, "dur": 0.212,
    "args": {
      "External id": 2482769,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034170.554, "dur": 0.464,
    "args": {
      "External id": 2482770,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034171.769, "dur": 0.594,
    "args": {
      "External id": 2482771,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034173.264, "dur": 0.465,
    "args": {
      "External id": 2482772,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034174.466, "dur": 0.433,
    "args": {
      "External id": 2482773,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034175.667, "dur": 0.420,
    "args": {
      "External id": 2482774,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034176.816, "dur": 0.439,
    "args": {
      "External id": 2482775,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034177.977, "dur": 0.452,
    "args": {
      "External id": 2482776,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034181.316, "dur": 0.212,
    "args": {
      "External id": 2482777,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034182.869, "dur": 0.180,
    "args": {
      "External id": 2482778,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034184.187, "dur": 0.542,
    "args": {
      "External id": 2482779,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034185.847, "dur": 0.236,
    "args": {
      "External id": 2482780,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034187.034, "dur": 0.406,
    "args": {
      "External id": 2482781,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034188.349, "dur": 0.417,
    "args": {
      "External id": 2482782,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034189.502, "dur": 0.218,
    "args": {
      "External id": 2482783,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034190.471, "dur": 0.222,
    "args": {
      "External id": 2482784,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034193.351, "dur": 0.218,
    "args": {
      "External id": 2482785,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034194.301, "dur": 0.243,
    "args": {
      "External id": 2482786,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034195.267, "dur": 0.215,
    "args": {
      "External id": 2482787,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034196.222, "dur": 0.218,
    "args": {
      "External id": 2482788,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034197.353, "dur": 0.219,
    "args": {
      "External id": 2482789,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034198.305, "dur": 0.220,
    "args": {
      "External id": 2482790,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034200.105, "dur": 0.210,
    "args": {
      "External id": 2482791,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034201.415, "dur": 0.222,
    "args": {
      "External id": 2482792,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034204.214, "dur": 0.474,
    "args": {
      "External id": 2482793,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034205.515, "dur": 0.454,
    "args": {
      "External id": 2482794,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034206.691, "dur": 0.443,
    "args": {
      "External id": 2482795,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034207.843, "dur": 0.455,
    "args": {
      "External id": 2482796,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034209.128, "dur": 0.229,
    "args": {
      "External id": 2482797,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034210.275, "dur": 0.219,
    "args": {
      "External id": 2482798,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034211.635, "dur": 0.213,
    "args": {
      "External id": 2482799,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034212.561, "dur": 0.218,
    "args": {
      "External id": 2482800,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034215.799, "dur": 0.210,
    "args": {
      "External id": 2482801,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034216.819, "dur": 0.217,
    "args": {
      "External id": 2482802,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034217.754, "dur": 0.230,
    "args": {
      "External id": 2482803,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034218.746, "dur": 0.232,
    "args": {
      "External id": 2482804,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034219.711, "dur": 0.213,
    "args": {
      "External id": 2482805,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034220.636, "dur": 0.437,
    "args": {
      "External id": 2482806,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034222.823, "dur": 0.424,
    "args": {
      "External id": 2482807,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034223.983, "dur": 0.442,
    "args": {
      "External id": 2482808,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034227.366, "dur": 0.216,
    "args": {
      "External id": 2482809,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034228.322, "dur": 0.423,
    "args": {
      "External id": 2482810,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034229.461, "dur": 0.421,
    "args": {
      "External id": 2482811,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034230.622, "dur": 0.418,
    "args": {
      "External id": 2482812,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034231.803, "dur": 0.454,
    "args": {
      "External id": 2482813,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034232.983, "dur": 0.430,
    "args": {
      "External id": 2482814,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034234.147, "dur": 0.423,
    "args": {
      "External id": 2482815,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034235.291, "dur": 0.220,
    "args": {
      "External id": 2482816,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034238.472, "dur": 0.216,
    "args": {
      "External id": 2482817,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034239.567, "dur": 0.240,
    "args": {
      "External id": 2482818,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034240.680, "dur": 0.220,
    "args": {
      "External id": 2482819,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034241.781, "dur": 0.221,
    "args": {
      "External id": 2482820,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034242.900, "dur": 0.220,
    "args": {
      "External id": 2482821,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034244.017, "dur": 0.220,
    "args": {
      "External id": 2482822,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034245.542, "dur": 0.211,
    "args": {
      "External id": 2482823,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034246.853, "dur": 0.218,
    "args": {
      "External id": 2482824,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034249.853, "dur": 0.216,
    "args": {
      "External id": 2482825,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034251.037, "dur": 0.229,
    "args": {
      "External id": 2482826,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034251.999, "dur": 0.215,
    "args": {
      "External id": 2482827,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034252.941, "dur": 0.221,
    "args": {
      "External id": 2482828,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034253.875, "dur": 0.214,
    "args": {
      "External id": 2482829,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034254.833, "dur": 0.218,
    "args": {
      "External id": 2482830,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034255.824, "dur": 0.211,
    "args": {
      "External id": 2482831,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034256.766, "dur": 0.218,
    "args": {
      "External id": 2482832,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034259.973, "dur": 0.218,
    "args": {
      "External id": 2482833,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034260.907, "dur": 0.234,
    "args": {
      "External id": 2482834,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034261.869, "dur": 0.214,
    "args": {
      "External id": 2482835,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034262.800, "dur": 0.219,
    "args": {
      "External id": 2482836,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034263.735, "dur": 0.213,
    "args": {
      "External id": 2482837,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034264.828, "dur": 0.220,
    "args": {
      "External id": 2482838,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034265.759, "dur": 0.216,
    "args": {
      "External id": 2482839,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034266.915, "dur": 0.219,
    "args": {
      "External id": 2482840,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034270.071, "dur": 0.208,
    "args": {
      "External id": 2482841,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034271.182, "dur": 0.443,
    "args": {
      "External id": 2482842,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034273.104, "dur": 0.175,
    "args": {
      "External id": 2482843,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034274.008, "dur": 0.449,
    "args": {
      "External id": 2482844,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034275.170, "dur": 0.447,
    "args": {
      "External id": 2482845,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034276.424, "dur": 0.433,
    "args": {
      "External id": 2482846,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034277.601, "dur": 0.401,
    "args": {
      "External id": 2482847,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034278.715, "dur": 0.413,
    "args": {
      "External id": 2482848,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034281.792, "dur": 0.210,
    "args": {
      "External id": 2482849,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034282.711, "dur": 0.397,
    "args": {
      "External id": 2482850,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034283.826, "dur": 0.412,
    "args": {
      "External id": 2482851,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034284.955, "dur": 0.376,
    "args": {
      "External id": 2482852,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034286.048, "dur": 0.213,
    "args": {
      "External id": 2482853,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034286.992, "dur": 0.216,
    "args": {
      "External id": 2482854,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034287.998, "dur": 0.212,
    "args": {
      "External id": 2482855,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034288.934, "dur": 0.215,
    "args": {
      "External id": 2482856,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034291.803, "dur": 0.213,
    "args": {
      "External id": 2482857,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034292.911, "dur": 0.216,
    "args": {
      "External id": 2482858,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034294.038, "dur": 0.215,
    "args": {
      "External id": 2482859,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034295.170, "dur": 0.218,
    "args": {
      "External id": 2482860,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034296.297, "dur": 0.211,
    "args": {
      "External id": 2482861,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034297.384, "dur": 0.219,
    "args": {
      "External id": 2482862,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034298.748, "dur": 0.224,
    "args": {
      "External id": 2482863,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034299.892, "dur": 0.217,
    "args": {
      "External id": 2482864,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034302.884, "dur": 0.214,
    "args": {
      "External id": 2482865,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034304.004, "dur": 0.218,
    "args": {
      "External id": 2482866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034305.119, "dur": 0.243,
    "args": {
      "External id": 2482867,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034306.260, "dur": 0.224,
    "args": {
      "External id": 2482868,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034307.200, "dur": 0.212,
    "args": {
      "External id": 2482869,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034308.120, "dur": 0.221,
    "args": {
      "External id": 2482870,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034309.102, "dur": 0.244,
    "args": {
      "External id": 2482871,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034310.054, "dur": 0.239,
    "args": {
      "External id": 2482872,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034314.696, "dur": 0.229,
    "args": {
      "External id": 2482873,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034315.652, "dur": 0.208,
    "args": {
      "External id": 2482874,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034316.610, "dur": 0.209,
    "args": {
      "External id": 2482875,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034317.533, "dur": 0.218,
    "args": {
      "External id": 2482876,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034318.464, "dur": 0.212,
    "args": {
      "External id": 2482877,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034319.442, "dur": 0.214,
    "args": {
      "External id": 2482878,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034320.373, "dur": 0.214,
    "args": {
      "External id": 2482879,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034321.336, "dur": 0.214,
    "args": {
      "External id": 2482880,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034324.084, "dur": 0.205,
    "args": {
      "External id": 2482881,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034325.025, "dur": 0.217,
    "args": {
      "External id": 2482882,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034326.004, "dur": 0.212,
    "args": {
      "External id": 2482883,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034326.926, "dur": 0.217,
    "args": {
      "External id": 2482884,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034327.862, "dur": 0.213,
    "args": {
      "External id": 2482885,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034328.782, "dur": 0.219,
    "args": {
      "External id": 2482886,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034330.057, "dur": 0.215,
    "args": {
      "External id": 2482887,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034330.989, "dur": 0.220,
    "args": {
      "External id": 2482888,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034333.521, "dur": 0.217,
    "args": {
      "External id": 2482889,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034334.463, "dur": 0.216,
    "args": {
      "External id": 2482890,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034335.450, "dur": 0.214,
    "args": {
      "External id": 2482891,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034336.416, "dur": 0.221,
    "args": {
      "External id": 2482892,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034337.355, "dur": 0.212,
    "args": {
      "External id": 2482893,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034338.289, "dur": 0.214,
    "args": {
      "External id": 2482894,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034339.221, "dur": 0.215,
    "args": {
      "External id": 2482895,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034340.147, "dur": 0.442,
    "args": {
      "External id": 2482896,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034343.183, "dur": 0.229,
    "args": {
      "External id": 2482897,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034344.312, "dur": 0.414,
    "args": {
      "External id": 2482898,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034345.623, "dur": 0.416,
    "args": {
      "External id": 2482899,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034346.918, "dur": 0.414,
    "args": {
      "External id": 2482900,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034348.217, "dur": 0.449,
    "args": {
      "External id": 2482901,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034349.685, "dur": 0.362,
    "args": {
      "External id": 2482902,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034350.758, "dur": 0.421,
    "args": {
      "External id": 2482903,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034352.015, "dur": 0.469,
    "args": {
      "External id": 2482904,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034355.123, "dur": 0.218,
    "args": {
      "External id": 2482905,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034356.268, "dur": 0.428,
    "args": {
      "External id": 2482906,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034357.710, "dur": 0.427,
    "args": {
      "External id": 2482907,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034359.048, "dur": 0.215,
    "args": {
      "External id": 2482908,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034360.200, "dur": 0.214,
    "args": {
      "External id": 2482909,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034361.412, "dur": 0.222,
    "args": {
      "External id": 2482910,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034362.499, "dur": 0.214,
    "args": {
      "External id": 2482911,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034363.615, "dur": 0.219,
    "args": {
      "External id": 2482912,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034365.994, "dur": 0.217,
    "args": {
      "External id": 2482913,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034366.924, "dur": 0.220,
    "args": {
      "External id": 2482914,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034368.355, "dur": 0.212,
    "args": {
      "External id": 2482915,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034369.283, "dur": 0.224,
    "args": {
      "External id": 2482916,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034370.215, "dur": 0.212,
    "args": {
      "External id": 2482917,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034371.180, "dur": 0.447,
    "args": {
      "External id": 2482918,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034372.335, "dur": 0.428,
    "args": {
      "External id": 2482919,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034373.492, "dur": 0.579,
    "args": {
      "External id": 2482920,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034377.212, "dur": 0.217,
    "args": {
      "External id": 2482921,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034378.193, "dur": 0.426,
    "args": {
      "External id": 2482922,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034379.327, "dur": 0.428,
    "args": {
      "External id": 2482923,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034380.460, "dur": 0.415,
    "args": {
      "External id": 2482924,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034381.710, "dur": 0.430,
    "args": {
      "External id": 2482925,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034382.843, "dur": 0.443,
    "args": {
      "External id": 2482926,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034384.000, "dur": 0.410,
    "args": {
      "External id": 2482927,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034385.112, "dur": 0.221,
    "args": {
      "External id": 2482928,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034388.485, "dur": 0.432,
    "args": {
      "External id": 2482929,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034389.664, "dur": 0.462,
    "args": {
      "External id": 2482930,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034390.839, "dur": 0.211,
    "args": {
      "External id": 2482931,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034391.758, "dur": 0.221,
    "args": {
      "External id": 2482932,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034392.853, "dur": 0.212,
    "args": {
      "External id": 2482933,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034393.959, "dur": 0.221,
    "args": {
      "External id": 2482934,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034395.216, "dur": 0.207,
    "args": {
      "External id": 2482935,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034396.130, "dur": 0.217,
    "args": {
      "External id": 2482936,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034399.040, "dur": 0.210,
    "args": {
      "External id": 2482937,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034399.969, "dur": 0.217,
    "args": {
      "External id": 2482938,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034400.928, "dur": 0.471,
    "args": {
      "External id": 2482939,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034402.105, "dur": 0.541,
    "args": {
      "External id": 2482940,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034403.349, "dur": 0.214,
    "args": {
      "External id": 2482941,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034404.268, "dur": 0.219,
    "args": {
      "External id": 2482942,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034405.268, "dur": 0.211,
    "args": {
      "External id": 2482943,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034406.178, "dur": 0.216,
    "args": {
      "External id": 2482944,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034408.852, "dur": 0.215,
    "args": {
      "External id": 2482945,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034409.770, "dur": 0.215,
    "args": {
      "External id": 2482946,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034411.255, "dur": 0.215,
    "args": {
      "External id": 2482947,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034412.178, "dur": 0.216,
    "args": {
      "External id": 2482948,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034413.357, "dur": 0.211,
    "args": {
      "External id": 2482949,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034414.446, "dur": 0.215,
    "args": {
      "External id": 2482950,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034415.618, "dur": 0.219,
    "args": {
      "External id": 2482951,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034416.792, "dur": 0.219,
    "args": {
      "External id": 2482952,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034419.834, "dur": 0.217,
    "args": {
      "External id": 2482953,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034420.980, "dur": 0.220,
    "args": {
      "External id": 2482954,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034422.111, "dur": 0.212,
    "args": {
      "External id": 2482955,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034423.054, "dur": 0.217,
    "args": {
      "External id": 2482956,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034423.982, "dur": 0.212,
    "args": {
      "External id": 2482957,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034425.091, "dur": 0.218,
    "args": {
      "External id": 2482958,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034426.230, "dur": 0.211,
    "args": {
      "External id": 2482959,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034427.173, "dur": 0.221,
    "args": {
      "External id": 2482960,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034429.764, "dur": 0.214,
    "args": {
      "External id": 2482961,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034430.684, "dur": 0.215,
    "args": {
      "External id": 2482962,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034431.654, "dur": 0.209,
    "args": {
      "External id": 2482963,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034432.596, "dur": 0.210,
    "args": {
      "External id": 2482964,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034433.511, "dur": 0.207,
    "args": {
      "External id": 2482965,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034434.416, "dur": 0.211,
    "args": {
      "External id": 2482966,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034435.331, "dur": 0.209,
    "args": {
      "External id": 2482967,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034436.244, "dur": 0.211,
    "args": {
      "External id": 2482968,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034439.743, "dur": 0.207,
    "args": {
      "External id": 2482969,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034444.983, "dur": 0.423,
    "args": {
      "External id": 2482970,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034446.290, "dur": 0.363,
    "args": {
      "External id": 2482971,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034447.427, "dur": 0.550,
    "args": {
      "External id": 2482972,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034448.718, "dur": 0.402,
    "args": {
      "External id": 2482973,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034449.943, "dur": 0.440,
    "args": {
      "External id": 2482974,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034451.130, "dur": 0.408,
    "args": {
      "External id": 2482975,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034452.317, "dur": 0.448,
    "args": {
      "External id": 2482976,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034455.049, "dur": 0.209,
    "args": {
      "External id": 2482977,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034455.967, "dur": 0.387,
    "args": {
      "External id": 2482978,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034457.076, "dur": 0.214,
    "args": {
      "External id": 2482979,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034458.014, "dur": 0.220,
    "args": {
      "External id": 2482980,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034458.973, "dur": 0.215,
    "args": {
      "External id": 2482981,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034460.051, "dur": 0.218,
    "args": {
      "External id": 2482982,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034461.404, "dur": 0.233,
    "args": {
      "External id": 2482983,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034462.347, "dur": 0.219,
    "args": {
      "External id": 2482984,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034465.164, "dur": 0.218,
    "args": {
      "External id": 2482985,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034466.094, "dur": 0.214,
    "args": {
      "External id": 2482986,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034468.669, "dur": 0.218,
    "args": {
      "External id": 2482987,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034469.627, "dur": 0.207,
    "args": {
      "External id": 2482988,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034470.561, "dur": 0.205,
    "args": {
      "External id": 2482989,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034471.491, "dur": 0.219,
    "args": {
      "External id": 2482990,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034472.419, "dur": 0.478,
    "args": {
      "External id": 2482991,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034473.784, "dur": 0.437,
    "args": {
      "External id": 2482992,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034477.022, "dur": 0.219,
    "args": {
      "External id": 2482993,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034478.144, "dur": 0.385,
    "args": {
      "External id": 2482994,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034479.264, "dur": 0.385,
    "args": {
      "External id": 2482995,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034480.623, "dur": 0.412,
    "args": {
      "External id": 2482996,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034481.742, "dur": 0.507,
    "args": {
      "External id": 2482997,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034483.122, "dur": 0.414,
    "args": {
      "External id": 2482998,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034484.259, "dur": 0.421,
    "args": {
      "External id": 2482999,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034485.616, "dur": 0.459,
    "args": {
      "External id": 2483000,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034488.810, "dur": 0.216,
    "args": {
      "External id": 2483001,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034489.749, "dur": 0.222,
    "args": {
      "External id": 2483002,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034490.679, "dur": 0.212,
    "args": {
      "External id": 2483003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034491.603, "dur": 0.215,
    "args": {
      "External id": 2483004,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336760, "tid": 1336760,
    "ts": 1514658034562.138, "dur": 1706.606,
    "args": {
      "External id": 2483005,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035027.588, "dur": 1160.823,
    "args": {
      "External id": 2483006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035036.255, "dur": 7.775,
    "args": {
      "External id": 2483007,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035040.188, "dur": 3.166,
    "args": {
      "External id": 2483008,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035044.643, "dur": 3.704,
    "args": {
      "External id": 2483009,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035045.608, "dur": 2.557,
    "args": {
      "External id": 2483010,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035048.751, "dur": 3.930,
    "args": {
      "External id": 2483011,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035051.033, "dur": 1.352,
    "args": {
      "External id": 2483012,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035052.973, "dur": 1.890,
    "args": {
      "External id": 2483013,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035053.614, "dur": 1.180,
    "args": {
      "External id": 2483014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035057.029, "dur": 4.417,
    "args": {
      "External id": 2483015,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035060.766, "dur": 0.608,
    "args": {
      "External id": 2483016,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035061.737, "dur": 1.872,
    "args": {
      "External id": 2483017,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035062.587, "dur": 0.956,
    "args": {
      "External id": 2483018,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035064.093, "dur": 1.592,
    "args": {
      "External id": 2483019,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035064.515, "dur": 1.102,
    "args": {
      "External id": 2483020,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035066.313, "dur": 3.754,
    "args": {
      "External id": 2483021,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035068.994, "dur": 1.005,
    "args": {
      "External id": 2483022,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035070.633, "dur": 1.798,
    "args": {
      "External id": 2483023,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035071.591, "dur": 0.779,
    "args": {
      "External id": 2483024,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035072.860, "dur": 3.181,
    "args": {
      "External id": 2483025,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035073.453, "dur": 2.501,
    "args": {
      "External id": 2483026,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035076.336, "dur": 3.276,
    "args": {
      "External id": 2483027,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035078.688, "dur": 0.860,
    "args": {
      "External id": 2483028,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035079.865, "dur": 1.284,
    "args": {
      "External id": 2483029,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035080.230, "dur": 0.853,
    "args": {
      "External id": 2483030,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035083.436, "dur": 4.858,
    "args": {
      "External id": 2483031,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035087.812, "dur": 0.411,
    "args": {
      "External id": 2483032,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035088.819, "dur": 1.981,
    "args": {
      "External id": 2483033,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035089.661, "dur": 1.078,
    "args": {
      "External id": 2483034,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035091.432, "dur": 1.816,
    "args": {
      "External id": 2483035,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035092.245, "dur": 0.936,
    "args": {
      "External id": 2483036,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035093.677, "dur": 3.213,
    "args": {
      "External id": 2483037,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035096.059, "dur": 0.756,
    "args": {
      "External id": 2483038,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035097.343, "dur": 1.623,
    "args": {
      "External id": 2483039,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035098.145, "dur": 0.759,
    "args": {
      "External id": 2483040,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035099.214, "dur": 3.080,
    "args": {
      "External id": 2483041,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035099.727, "dur": 2.499,
    "args": {
      "External id": 2483042,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035102.546, "dur": 2.255,
    "args": {
      "External id": 2483043,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035104.200, "dur": 0.539,
    "args": {
      "External id": 2483044,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035105.049, "dur": 1.016,
    "args": {
      "External id": 2483045,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035105.408, "dur": 0.590,
    "args": {
      "External id": 2483046,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035108.644, "dur": 5.010,
    "args": {
      "External id": 2483047,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035112.785, "dur": 0.801,
    "args": {
      "External id": 2483048,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035114.195, "dur": 2.136,
    "args": {
      "External id": 2483049,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035115.151, "dur": 1.120,
    "args": {
      "External id": 2483050,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035116.760, "dur": 1.750,
    "args": {
      "External id": 2483051,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035117.550, "dur": 0.892,
    "args": {
      "External id": 2483052,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035118.976, "dur": 3.986,
    "args": {
      "External id": 2483053,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035122.072, "dur": 0.827,
    "args": {
      "External id": 2483054,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035123.397, "dur": 1.723,
    "args": {
      "External id": 2483055,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035124.286, "dur": 0.773,
    "args": {
      "External id": 2483056,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035125.545, "dur": 3.532,
    "args": {
      "External id": 2483057,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035126.310, "dur": 2.696,
    "args": {
      "External id": 2483058,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035129.508, "dur": 3.224,
    "args": {
      "External id": 2483059,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035131.804, "dur": 0.866,
    "args": {
      "External id": 2483060,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035133.350, "dur": 1.152,
    "args": {
      "External id": 2483061,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035133.700, "dur": 0.734,
    "args": {
      "External id": 2483062,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035136.721, "dur": 4.467,
    "args": {
      "External id": 2483063,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035140.485, "dur": 0.638,
    "args": {
      "External id": 2483064,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035141.432, "dur": 1.293,
    "args": {
      "External id": 2483065,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035142.028, "dur": 0.636,
    "args": {
      "External id": 2483066,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035142.965, "dur": 1.627,
    "args": {
      "External id": 2483067,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035143.571, "dur": 0.955,
    "args": {
      "External id": 2483068,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035145.022, "dur": 3.794,
    "args": {
      "External id": 2483069,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035148.002, "dur": 0.751,
    "args": {
      "External id": 2483070,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035149.249, "dur": 1.613,
    "args": {
      "External id": 2483071,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035150.036, "dur": 0.762,
    "args": {
      "External id": 2483072,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035151.329, "dur": 3.473,
    "args": {
      "External id": 2483073,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035152.157, "dur": 2.572,
    "args": {
      "External id": 2483074,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035155.067, "dur": 4.506,
    "args": {
      "External id": 2483075,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035158.888, "dur": 0.619,
    "args": {
      "External id": 2483076,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035159.972, "dur": 2.999,
    "args": {
      "External id": 2483077,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035160.558, "dur": 2.336,
    "args": {
      "External id": 2483078,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035165.167, "dur": 3.114,
    "args": {
      "External id": 2483079,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035167.128, "dur": 1.088,
    "args": {
      "External id": 2483080,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035168.873, "dur": 1.878,
    "args": {
      "External id": 2483081,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035169.649, "dur": 1.031,
    "args": {
      "External id": 2483082,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035170.988, "dur": 3.594,
    "args": {
      "External id": 2483083,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035173.743, "dur": 0.778,
    "args": {
      "External id": 2483084,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035174.825, "dur": 2.963,
    "args": {
      "External id": 2483085,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035176.874, "dur": 0.844,
    "args": {
      "External id": 2483086,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035178.076, "dur": 1.560,
    "args": {
      "External id": 2483087,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035178.467, "dur": 1.102,
    "args": {
      "External id": 2483088,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035180.394, "dur": 3.679,
    "args": {
      "External id": 2483089,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035183.186, "dur": 0.823,
    "args": {
      "External id": 2483090,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035184.510, "dur": 3.550,
    "args": {
      "External id": 2483091,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035187.107, "dur": 0.890,
    "args": {
      "External id": 2483092,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035188.647, "dur": 2.897,
    "args": {
      "External id": 2483093,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035189.239, "dur": 2.042,
    "args": {
      "External id": 2483094,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035193.718, "dur": 2.181,
    "args": {
      "External id": 2483095,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035195.072, "dur": 0.764,
    "args": {
      "External id": 2483096,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035196.358, "dur": 1.147,
    "args": {
      "External id": 2483097,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035196.712, "dur": 0.725,
    "args": {
      "External id": 2483098,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035198.006, "dur": 3.560,
    "args": {
      "External id": 2483099,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035200.519, "dur": 0.986,
    "args": {
      "External id": 2483100,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035201.986, "dur": 2.135,
    "args": {
      "External id": 2483101,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035203.100, "dur": 0.961,
    "args": {
      "External id": 2483102,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035204.566, "dur": 2.003,
    "args": {
      "External id": 2483103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035205.624, "dur": 0.882,
    "args": {
      "External id": 2483104,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035206.813, "dur": 3.906,
    "args": {
      "External id": 2483105,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035209.818, "dur": 0.839,
    "args": {
      "External id": 2483106,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035210.958, "dur": 2.104,
    "args": {
      "External id": 2483107,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035212.145, "dur": 0.854,
    "args": {
      "External id": 2483108,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035213.292, "dur": 3.704,
    "args": {
      "External id": 2483109,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035214.311, "dur": 2.438,
    "args": {
      "External id": 2483110,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035219.320, "dur": 1.630,
    "args": {
      "External id": 2483111,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035220.032, "dur": 0.857,
    "args": {
      "External id": 2483112,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035221.355, "dur": 1.611,
    "args": {
      "External id": 2483113,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035222.221, "dur": 0.678,
    "args": {
      "External id": 2483114,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035223.248, "dur": 3.549,
    "args": {
      "External id": 2483115,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035225.665, "dur": 1.066,
    "args": {
      "External id": 2483116,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035227.205, "dur": 2.108,
    "args": {
      "External id": 2483117,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035228.545, "dur": 0.707,
    "args": {
      "External id": 2483118,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035229.584, "dur": 1.409,
    "args": {
      "External id": 2483119,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035230.236, "dur": 0.689,
    "args": {
      "External id": 2483120,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035231.393, "dur": 4.148,
    "args": {
      "External id": 2483121,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035234.783, "dur": 0.697,
    "args": {
      "External id": 2483122,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035236.010, "dur": 1.756,
    "args": {
      "External id": 2483123,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035236.902, "dur": 0.804,
    "args": {
      "External id": 2483124,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035237.989, "dur": 3.500,
    "args": {
      "External id": 2483125,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035239.033, "dur": 2.384,
    "args": {
      "External id": 2483126,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035243.668, "dur": 1.489,
    "args": {
      "External id": 2483127,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035244.516, "dur": 0.576,
    "args": {
      "External id": 2483128,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035245.572, "dur": 2.242,
    "args": {
      "External id": 2483129,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035247.045, "dur": 0.701,
    "args": {
      "External id": 2483130,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035248.226, "dur": 3.830,
    "args": {
      "External id": 2483131,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035251.363, "dur": 0.632,
    "args": {
      "External id": 2483132,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035252.304, "dur": 1.968,
    "args": {
      "External id": 2483133,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035253.599, "dur": 0.613,
    "args": {
      "External id": 2483134,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035254.516, "dur": 1.367,
    "args": {
      "External id": 2483135,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035255.149, "dur": 0.667,
    "args": {
      "External id": 2483136,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035256.326, "dur": 3.962,
    "args": {
      "External id": 2483137,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035259.590, "dur": 0.638,
    "args": {
      "External id": 2483138,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035260.803, "dur": 1.528,
    "args": {
      "External id": 2483139,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035261.637, "dur": 0.629,
    "args": {
      "External id": 2483140,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035262.582, "dur": 3.662,
    "args": {
      "External id": 2483141,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035263.960, "dur": 2.211,
    "args": {
      "External id": 2483142,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035268.440, "dur": 1.342,
    "args": {
      "External id": 2483143,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035269.106, "dur": 0.614,
    "args": {
      "External id": 2483144,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035270.176, "dur": 2.142,
    "args": {
      "External id": 2483145,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035271.488, "dur": 0.762,
    "args": {
      "External id": 2483146,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035272.755, "dur": 4.588,
    "args": {
      "External id": 2483147,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035276.316, "dur": 0.964,
    "args": {
      "External id": 2483148,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035277.765, "dur": 2.032,
    "args": {
      "External id": 2483149,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035278.834, "dur": 0.901,
    "args": {
      "External id": 2483150,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035280.229, "dur": 2.194,
    "args": {
      "External id": 2483151,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035281.309, "dur": 1.047,
    "args": {
      "External id": 2483152,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035282.648, "dur": 4.638,
    "args": {
      "External id": 2483153,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035286.369, "dur": 0.855,
    "args": {
      "External id": 2483154,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035287.716, "dur": 1.795,
    "args": {
      "External id": 2483155,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035288.598, "dur": 0.851,
    "args": {
      "External id": 2483156,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035289.744, "dur": 3.501,
    "args": {
      "External id": 2483157,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035290.943, "dur": 2.232,
    "args": {
      "External id": 2483158,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035295.317, "dur": 1.871,
    "args": {
      "External id": 2483159,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035296.175, "dur": 0.953,
    "args": {
      "External id": 2483160,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035297.442, "dur": 2.214,
    "args": {
      "External id": 2483161,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035298.705, "dur": 0.884,
    "args": {
      "External id": 2483162,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035300.087, "dur": 4.650,
    "args": {
      "External id": 2483163,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035303.787, "dur": 0.888,
    "args": {
      "External id": 2483164,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035305.195, "dur": 2.189,
    "args": {
      "External id": 2483165,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035306.525, "dur": 0.796,
    "args": {
      "External id": 2483166,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035307.825, "dur": 1.856,
    "args": {
      "External id": 2483167,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035308.628, "dur": 0.986,
    "args": {
      "External id": 2483168,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035310.120, "dur": 3.953,
    "args": {
      "External id": 2483169,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035313.230, "dur": 0.778,
    "args": {
      "External id": 2483170,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035314.314, "dur": 2.289,
    "args": {
      "External id": 2483171,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035315.352, "dur": 1.191,
    "args": {
      "External id": 2483172,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035317.053, "dur": 3.851,
    "args": {
      "External id": 2483173,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035318.395, "dur": 2.266,
    "args": {
      "External id": 2483174,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035323.220, "dur": 2.276,
    "args": {
      "External id": 2483175,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035324.285, "dur": 1.150,
    "args": {
      "External id": 2483176,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035325.914, "dur": 2.646,
    "args": {
      "External id": 2483177,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035327.274, "dur": 1.219,
    "args": {
      "External id": 2483178,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035328.827, "dur": 4.234,
    "args": {
      "External id": 2483179,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035332.160, "dur": 0.838,
    "args": {
      "External id": 2483180,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035333.304, "dur": 2.010,
    "args": {
      "External id": 2483181,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035334.402, "dur": 0.849,
    "args": {
      "External id": 2483182,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035335.755, "dur": 2.118,
    "args": {
      "External id": 2483183,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035336.922, "dur": 0.882,
    "args": {
      "External id": 2483184,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035338.293, "dur": 4.312,
    "args": {
      "External id": 2483185,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035341.720, "dur": 0.819,
    "args": {
      "External id": 2483186,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035343.040, "dur": 1.779,
    "args": {
      "External id": 2483187,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035344.191, "dur": 0.562,
    "args": {
      "External id": 2483188,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035345.427, "dur": 3.552,
    "args": {
      "External id": 2483189,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035346.531, "dur": 2.380,
    "args": {
      "External id": 2483190,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035351.306, "dur": 1.958,
    "args": {
      "External id": 2483191,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035352.395, "dur": 0.800,
    "args": {
      "External id": 2483192,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035353.489, "dur": 2.195,
    "args": {
      "External id": 2483193,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035354.912, "dur": 0.700,
    "args": {
      "External id": 2483194,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035356.319, "dur": 3.617,
    "args": {
      "External id": 2483195,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035359.274, "dur": 0.594,
    "args": {
      "External id": 2483196,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035360.479, "dur": 1.829,
    "args": {
      "External id": 2483197,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035361.555, "dur": 0.694,
    "args": {
      "External id": 2483198,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035362.938, "dur": 1.787,
    "args": {
      "External id": 2483199,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035363.811, "dur": 0.848,
    "args": {
      "External id": 2483200,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035395.337, "dur": 3.799,
    "args": {
      "External id": 2489345,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035398.619, "dur": 0.446,
    "args": {
      "External id": 2489346,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035399.425, "dur": 1.749,
    "args": {
      "External id": 2489347,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035400.363, "dur": 0.738,
    "args": {
      "External id": 2489348,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035401.606, "dur": 3.478,
    "args": {
      "External id": 2489349,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035402.752, "dur": 2.149,
    "args": {
      "External id": 2489350,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035407.425, "dur": 1.167,
    "args": {
      "External id": 2489351,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035408.095, "dur": 0.428,
    "args": {
      "External id": 2489352,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035408.850, "dur": 1.682,
    "args": {
      "External id": 2489353,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035409.929, "dur": 0.533,
    "args": {
      "External id": 2489354,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035410.844, "dur": 3.297,
    "args": {
      "External id": 2489355,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035413.657, "dur": 0.414,
    "args": {
      "External id": 2489356,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035414.380, "dur": 1.666,
    "args": {
      "External id": 2489357,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035415.348, "dur": 0.634,
    "args": {
      "External id": 2489358,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035416.292, "dur": 1.664,
    "args": {
      "External id": 2489359,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035417.111, "dur": 0.776,
    "args": {
      "External id": 2489360,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035418.233, "dur": 3.689,
    "args": {
      "External id": 2489361,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035421.101, "dur": 0.754,
    "args": {
      "External id": 2489362,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035422.163, "dur": 1.473,
    "args": {
      "External id": 2489363,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035422.879, "dur": 0.695,
    "args": {
      "External id": 2489364,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035423.972, "dur": 3.658,
    "args": {
      "External id": 2489365,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035425.061, "dur": 2.495,
    "args": {
      "External id": 2489366,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035429.702, "dur": 1.820,
    "args": {
      "External id": 2489367,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035430.561, "dur": 0.899,
    "args": {
      "External id": 2489368,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035431.748, "dur": 2.107,
    "args": {
      "External id": 2489369,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035433.082, "dur": 0.706,
    "args": {
      "External id": 2489370,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035434.107, "dur": 3.696,
    "args": {
      "External id": 2489371,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035437.055, "dur": 0.686,
    "args": {
      "External id": 2489372,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035438.075, "dur": 1.855,
    "args": {
      "External id": 2489373,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035439.205, "dur": 0.662,
    "args": {
      "External id": 2489374,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035440.289, "dur": 1.519,
    "args": {
      "External id": 2489375,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035441.011, "dur": 0.731,
    "args": {
      "External id": 2489376,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035442.268, "dur": 3.928,
    "args": {
      "External id": 2489377,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035445.364, "dur": 0.770,
    "args": {
      "External id": 2489378,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035446.477, "dur": 1.583,
    "args": {
      "External id": 2489379,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035447.212, "dur": 0.781,
    "args": {
      "External id": 2489380,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035448.290, "dur": 3.286,
    "args": {
      "External id": 2489381,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035449.520, "dur": 1.984,
    "args": {
      "External id": 2489382,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035453.951, "dur": 1.492,
    "args": {
      "External id": 2489383,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035454.819, "dur": 0.560,
    "args": {
      "External id": 2489384,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035455.784, "dur": 2.043,
    "args": {
      "External id": 2489385,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035457.156, "dur": 0.604,
    "args": {
      "External id": 2489386,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035458.165, "dur": 3.885,
    "args": {
      "External id": 2489387,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035461.159, "dur": 0.822,
    "args": {
      "External id": 2489388,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035462.314, "dur": 1.753,
    "args": {
      "External id": 2489389,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035463.400, "dur": 0.607,
    "args": {
      "External id": 2489390,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035464.314, "dur": 1.678,
    "args": {
      "External id": 2489391,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035465.217, "dur": 0.706,
    "args": {
      "External id": 2489392,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035466.240, "dur": 3.977,
    "args": {
      "External id": 2489393,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035469.536, "dur": 0.620,
    "args": {
      "External id": 2489394,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035470.481, "dur": 1.293,
    "args": {
      "External id": 2489395,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035471.172, "dur": 0.538,
    "args": {
      "External id": 2489396,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035471.994, "dur": 3.251,
    "args": {
      "External id": 2489397,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035473.041, "dur": 2.131,
    "args": {
      "External id": 2489398,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035477.550, "dur": 1.520,
    "args": {
      "External id": 2489399,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035478.219, "dur": 0.786,
    "args": {
      "External id": 2489400,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035479.333, "dur": 1.993,
    "args": {
      "External id": 2489401,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035480.411, "dur": 0.848,
    "args": {
      "External id": 2489402,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035481.566, "dur": 3.128,
    "args": {
      "External id": 2489403,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035484.072, "dur": 0.560,
    "args": {
      "External id": 2489404,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035484.943, "dur": 1.644,
    "args": {
      "External id": 2489405,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035486.003, "dur": 0.520,
    "args": {
      "External id": 2489406,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035487.022, "dur": 1.690,
    "args": {
      "External id": 2489407,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035487.846, "dur": 0.796,
    "args": {
      "External id": 2489408,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035488.932, "dur": 3.447,
    "args": {
      "External id": 2489409,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035491.658, "dur": 0.655,
    "args": {
      "External id": 2489410,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035492.816, "dur": 1.463,
    "args": {
      "External id": 2489411,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035493.652, "dur": 0.561,
    "args": {
      "External id": 2489412,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035494.499, "dur": 2.999,
    "args": {
      "External id": 2489413,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035495.612, "dur": 1.813,
    "args": {
      "External id": 2489414,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035499.293, "dur": 1.533,
    "args": {
      "External id": 2489415,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035500.039, "dur": 0.725,
    "args": {
      "External id": 2489416,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035501.246, "dur": 2.132,
    "args": {
      "External id": 2489417,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035502.515, "dur": 0.796,
    "args": {
      "External id": 2489418,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035503.652, "dur": 3.379,
    "args": {
      "External id": 2489419,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035506.374, "dur": 0.588,
    "args": {
      "External id": 2489420,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035507.286, "dur": 2.131,
    "args": {
      "External id": 2489421,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035508.778, "dur": 0.573,
    "args": {
      "External id": 2489422,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035510.051, "dur": 1.611,
    "args": {
      "External id": 2489423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035510.869, "dur": 0.725,
    "args": {
      "External id": 2489424,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035511.880, "dur": 3.739,
    "args": {
      "External id": 2489425,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035514.800, "dur": 0.753,
    "args": {
      "External id": 2489426,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035515.857, "dur": 2.012,
    "args": {
      "External id": 2489427,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035516.881, "dur": 0.921,
    "args": {
      "External id": 2489428,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035518.090, "dur": 3.351,
    "args": {
      "External id": 2489429,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035519.330, "dur": 2.040,
    "args": {
      "External id": 2489430,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035523.412, "dur": 1.518,
    "args": {
      "External id": 2489431,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035524.296, "dur": 0.572,
    "args": {
      "External id": 2489432,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035525.167, "dur": 2.264,
    "args": {
      "External id": 2489433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035526.658, "dur": 0.704,
    "args": {
      "External id": 2489434,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035527.669, "dur": 17.426,
    "args": {
      "External id": 2489435,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035543.768, "dur": 0.876,
    "args": {
      "External id": 2489436,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035545.541, "dur": 2.185,
    "args": {
      "External id": 2489437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035546.936, "dur": 0.727,
    "args": {
      "External id": 2489438,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035548.021, "dur": 1.960,
    "args": {
      "External id": 2489439,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035548.886, "dur": 1.026,
    "args": {
      "External id": 2489440,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035550.389, "dur": 4.058,
    "args": {
      "External id": 2489441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035553.656, "dur": 0.731,
    "args": {
      "External id": 2489442,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035554.880, "dur": 1.869,
    "args": {
      "External id": 2489443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035556.024, "dur": 0.664,
    "args": {
      "External id": 2489444,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035557.023, "dur": 3.552,
    "args": {
      "External id": 2489445,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035558.211, "dur": 2.292,
    "args": {
      "External id": 2489446,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035562.629, "dur": 1.550,
    "args": {
      "External id": 2489447,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035563.520, "dur": 0.595,
    "args": {
      "External id": 2489448,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035564.572, "dur": 2.070,
    "args": {
      "External id": 2489449,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035565.907, "dur": 0.668,
    "args": {
      "External id": 2489450,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035567.075, "dur": 3.533,
    "args": {
      "External id": 2489451,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035570.048, "dur": 0.494,
    "args": {
      "External id": 2489452,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035570.858, "dur": 2.171,
    "args": {
      "External id": 2489453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035572.172, "dur": 0.796,
    "args": {
      "External id": 2489454,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035573.274, "dur": 1.662,
    "args": {
      "External id": 2489455,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035574.230, "dur": 0.640,
    "args": {
      "External id": 2489456,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035575.187, "dur": 3.775,
    "args": {
      "External id": 2489457,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035578.235, "dur": 0.667,
    "args": {
      "External id": 2489458,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035579.235, "dur": 1.266,
    "args": {
      "External id": 2489459,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035579.864, "dur": 0.576,
    "args": {
      "External id": 2489460,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035580.906, "dur": 3.684,
    "args": {
      "External id": 2489461,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035582.151, "dur": 2.281,
    "args": {
      "External id": 2489462,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035586.655, "dur": 1.497,
    "args": {
      "External id": 2489463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035587.298, "dur": 0.791,
    "args": {
      "External id": 2489464,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035588.378, "dur": 2.037,
    "args": {
      "External id": 2489465,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035589.631, "dur": 0.713,
    "args": {
      "External id": 2489466,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035590.667, "dur": 3.596,
    "args": {
      "External id": 2489467,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035593.237, "dur": 0.961,
    "args": {
      "External id": 2489468,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035594.527, "dur": 2.207,
    "args": {
      "External id": 2489469,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035595.869, "dur": 0.805,
    "args": {
      "External id": 2489470,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035596.977, "dur": 1.632,
    "args": {
      "External id": 2489471,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035597.881, "dur": 0.662,
    "args": {
      "External id": 2489472,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035598.889, "dur": 3.749,
    "args": {
      "External id": 2489473,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035601.984, "dur": 0.583,
    "args": {
      "External id": 2489474,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035603.073, "dur": 3.188,
    "args": {
      "External id": 2489475,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035605.436, "dur": 0.756,
    "args": {
      "External id": 2489476,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035606.702, "dur": 3.332,
    "args": {
      "External id": 2489477,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035607.973, "dur": 1.990,
    "args": {
      "External id": 2489478,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035612.190, "dur": 1.723,
    "args": {
      "External id": 2489479,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035613.146, "dur": 0.706,
    "args": {
      "External id": 2489480,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035614.140, "dur": 1.830,
    "args": {
      "External id": 2489481,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035615.282, "dur": 0.623,
    "args": {
      "External id": 2489482,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035616.406, "dur": 3.389,
    "args": {
      "External id": 2489483,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035619.191, "dur": 0.542,
    "args": {
      "External id": 2489484,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035620.062, "dur": 2.098,
    "args": {
      "External id": 2489485,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035621.397, "dur": 0.701,
    "args": {
      "External id": 2489486,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035622.404, "dur": 1.375,
    "args": {
      "External id": 2489487,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035623.061, "dur": 0.651,
    "args": {
      "External id": 2489488,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035624.071, "dur": 3.931,
    "args": {
      "External id": 2489489,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035627.301, "dur": 0.636,
    "args": {
      "External id": 2489490,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035628.256, "dur": 1.448,
    "args": {
      "External id": 2489491,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035629.057, "dur": 0.584,
    "args": {
      "External id": 2489492,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035629.998, "dur": 3.544,
    "args": {
      "External id": 2489493,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035631.200, "dur": 2.267,
    "args": {
      "External id": 2489494,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035635.966, "dur": 1.512,
    "args": {
      "External id": 2489495,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035636.621, "dur": 0.798,
    "args": {
      "External id": 2489496,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035637.795, "dur": 1.878,
    "args": {
      "External id": 2489497,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035638.907, "dur": 0.699,
    "args": {
      "External id": 2489498,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035639.917, "dur": 3.458,
    "args": {
      "External id": 2489499,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035642.629, "dur": 0.681,
    "args": {
      "External id": 2489500,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035643.623, "dur": 1.860,
    "args": {
      "External id": 2489501,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035644.844, "dur": 0.575,
    "args": {
      "External id": 2489502,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035645.921, "dur": 1.500,
    "args": {
      "External id": 2489503,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035646.721, "dur": 0.634,
    "args": {
      "External id": 2489504,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035647.646, "dur": 3.905,
    "args": {
      "External id": 2489505,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035650.984, "dur": 0.505,
    "args": {
      "External id": 2489506,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035651.988, "dur": 1.485,
    "args": {
      "External id": 2489507,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035652.842, "dur": 0.565,
    "args": {
      "External id": 2489508,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035653.897, "dur": 2.960,
    "args": {
      "External id": 2489509,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035654.781, "dur": 2.006,
    "args": {
      "External id": 2489510,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035659.071, "dur": 1.524,
    "args": {
      "External id": 2489511,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035659.958, "dur": 0.572,
    "args": {
      "External id": 2489512,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035660.905, "dur": 1.999,
    "args": {
      "External id": 2489513,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035662.179, "dur": 0.658,
    "args": {
      "External id": 2489514,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035663.157, "dur": 3.059,
    "args": {
      "External id": 2489515,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035665.639, "dur": 0.514,
    "args": {
      "External id": 2489516,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035666.464, "dur": 1.855,
    "args": {
      "External id": 2489517,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035667.669, "dur": 0.584,
    "args": {
      "External id": 2489518,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035668.565, "dur": 1.574,
    "args": {
      "External id": 2489519,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035669.402, "dur": 0.670,
    "args": {
      "External id": 2489520,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035670.632, "dur": 3.440,
    "args": {
      "External id": 2489521,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035673.415, "dur": 0.594,
    "args": {
      "External id": 2489522,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035674.423, "dur": 1.730,
    "args": {
      "External id": 2489523,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035675.501, "dur": 0.589,
    "args": {
      "External id": 2489524,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035676.449, "dur": 3.117,
    "args": {
      "External id": 2489525,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035677.354, "dur": 2.141,
    "args": {
      "External id": 2489526,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035681.635, "dur": 1.624,
    "args": {
      "External id": 2489527,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035682.469, "dur": 0.730,
    "args": {
      "External id": 2489528,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035683.522, "dur": 2.197,
    "args": {
      "External id": 2489529,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035684.923, "dur": 0.729,
    "args": {
      "External id": 2489530,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035685.962, "dur": 3.858,
    "args": {
      "External id": 2489531,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035689.037, "dur": 0.719,
    "args": {
      "External id": 2489532,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035690.067, "dur": 1.862,
    "args": {
      "External id": 2489533,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035691.303, "dur": 0.563,
    "args": {
      "External id": 2489534,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035692.172, "dur": 1.538,
    "args": {
      "External id": 2489535,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035693.102, "dur": 0.543,
    "args": {
      "External id": 2489536,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035694.008, "dur": 4.130,
    "args": {
      "External id": 2489537,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035697.490, "dur": 0.578,
    "args": {
      "External id": 2489538,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035698.496, "dur": 1.423,
    "args": {
      "External id": 2489539,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035699.198, "dur": 0.652,
    "args": {
      "External id": 2489540,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035700.167, "dur": 3.436,
    "args": {
      "External id": 2489541,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035701.335, "dur": 2.198,
    "args": {
      "External id": 2489542,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035705.658, "dur": 1.948,
    "args": {
      "External id": 2489543,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035706.422, "dur": 1.121,
    "args": {
      "External id": 2489544,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035707.826, "dur": 2.294,
    "args": {
      "External id": 2489545,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035709.205, "dur": 0.848,
    "args": {
      "External id": 2489546,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035710.886, "dur": 3.415,
    "args": {
      "External id": 2489547,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035713.510, "dur": 0.729,
    "args": {
      "External id": 2489548,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035714.547, "dur": 2.327,
    "args": {
      "External id": 2489549,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035715.969, "dur": 0.841,
    "args": {
      "External id": 2489550,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035717.114, "dur": 1.448,
    "args": {
      "External id": 2489551,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035717.754, "dur": 0.739,
    "args": {
      "External id": 2489552,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035718.845, "dur": 3.884,
    "args": {
      "External id": 2489553,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035722.074, "dur": 0.589,
    "args": {
      "External id": 2489554,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035722.973, "dur": 1.660,
    "args": {
      "External id": 2489555,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035723.900, "dur": 0.666,
    "args": {
      "External id": 2489556,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035725.049, "dur": 3.349,
    "args": {
      "External id": 2489557,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035726.136, "dur": 2.191,
    "args": {
      "External id": 2489558,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035730.752, "dur": 1.236,
    "args": {
      "External id": 2489559,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035731.448, "dur": 0.479,
    "args": {
      "External id": 2489560,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035732.409, "dur": 1.966,
    "args": {
      "External id": 2489561,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035733.518, "dur": 0.792,
    "args": {
      "External id": 2489562,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035734.807, "dur": 3.802,
    "args": {
      "External id": 2489563,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035738.110, "dur": 0.438,
    "args": {
      "External id": 2489564,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035738.905, "dur": 1.602,
    "args": {
      "External id": 2489565,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035739.919, "dur": 0.524,
    "args": {
      "External id": 2489566,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035740.920, "dur": 1.602,
    "args": {
      "External id": 2489567,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035741.794, "dur": 0.659,
    "args": {
      "External id": 2489568,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035742.988, "dur": 3.426,
    "args": {
      "External id": 2489569,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035745.756, "dur": 0.597,
    "args": {
      "External id": 2489570,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035746.887, "dur": 1.449,
    "args": {
      "External id": 2489571,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035747.533, "dur": 0.742,
    "args": {
      "External id": 2489572,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035748.588, "dur": 4.002,
    "args": {
      "External id": 2489573,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035749.693, "dur": 2.561,
    "args": {
      "External id": 2489574,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035755.136, "dur": 1.537,
    "args": {
      "External id": 2489575,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035755.843, "dur": 0.765,
    "args": {
      "External id": 2489576,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035757.498, "dur": 2.069,
    "args": {
      "External id": 2489577,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035758.794, "dur": 0.705,
    "args": {
      "External id": 2489578,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035759.807, "dur": 3.412,
    "args": {
      "External id": 2489579,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035762.570, "dur": 0.587,
    "args": {
      "External id": 2489580,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035763.532, "dur": 2.434,
    "args": {
      "External id": 2489581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035764.966, "dur": 0.936,
    "args": {
      "External id": 2489582,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035766.383, "dur": 1.420,
    "args": {
      "External id": 2489583,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035767.057, "dur": 0.677,
    "args": {
      "External id": 2489584,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035768.091, "dur": 3.842,
    "args": {
      "External id": 2489585,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035771.253, "dur": 0.617,
    "args": {
      "External id": 2489586,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035772.200, "dur": 1.312,
    "args": {
      "External id": 2489587,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035772.884, "dur": 0.564,
    "args": {
      "External id": 2489588,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035773.778, "dur": 3.761,
    "args": {
      "External id": 2489589,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035775.072, "dur": 2.300,
    "args": {
      "External id": 2489590,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035779.566, "dur": 1.375,
    "args": {
      "External id": 2489591,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035780.245, "dur": 0.636,
    "args": {
      "External id": 2489592,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035781.203, "dur": 1.899,
    "args": {
      "External id": 2489593,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035782.341, "dur": 0.693,
    "args": {
      "External id": 2489594,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035783.534, "dur": 3.456,
    "args": {
      "External id": 2489595,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035786.303, "dur": 0.622,
    "args": {
      "External id": 2489596,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035787.235, "dur": 2.032,
    "args": {
      "External id": 2489597,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035788.724, "dur": 0.481,
    "args": {
      "External id": 2489598,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035789.724, "dur": 1.814,
    "args": {
      "External id": 2489599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035790.591, "dur": 0.878,
    "args": {
      "External id": 2489600,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035792.098, "dur": 3.592,
    "args": {
      "External id": 2489601,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035795.090, "dur": 0.536,
    "args": {
      "External id": 2489602,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035795.947, "dur": 1.535,
    "args": {
      "External id": 2489603,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035796.830, "dur": 0.591,
    "args": {
      "External id": 2489604,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035797.705, "dur": 3.879,
    "args": {
      "External id": 2489605,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035798.981, "dur": 2.533,
    "args": {
      "External id": 2489606,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035804.043, "dur": 1.358,
    "args": {
      "External id": 2489607,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035804.726, "dur": 0.612,
    "args": {
      "External id": 2489608,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035805.665, "dur": 2.266,
    "args": {
      "External id": 2489609,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035807.066, "dur": 0.800,
    "args": {
      "External id": 2489610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035808.197, "dur": 3.646,
    "args": {
      "External id": 2489611,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035811.239, "dur": 0.544,
    "args": {
      "External id": 2489612,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035812.099, "dur": 2.014,
    "args": {
      "External id": 2489613,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035813.418, "dur": 0.633,
    "args": {
      "External id": 2489614,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035814.535, "dur": 1.351,
    "args": {
      "External id": 2489615,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035815.190, "dur": 0.628,
    "args": {
      "External id": 2489616,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035816.175, "dur": 4.022,
    "args": {
      "External id": 2489617,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035819.512, "dur": 0.624,
    "args": {
      "External id": 2489618,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035820.442, "dur": 1.914,
    "args": {
      "External id": 2489619,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035821.533, "dur": 0.586,
    "args": {
      "External id": 2489620,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035822.574, "dur": 3.924,
    "args": {
      "External id": 2489621,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035824.021, "dur": 2.407,
    "args": {
      "External id": 2489622,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035828.736, "dur": 1.336,
    "args": {
      "External id": 2489623,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035829.391, "dur": 0.619,
    "args": {
      "External id": 2489624,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035830.296, "dur": 1.925,
    "args": {
      "External id": 2489625,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035831.478, "dur": 0.676,
    "args": {
      "External id": 2489626,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035832.462, "dur": 3.406,
    "args": {
      "External id": 2489627,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035834.963, "dur": 0.844,
    "args": {
      "External id": 2489628,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035836.134, "dur": 2.158,
    "args": {
      "External id": 2489629,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035837.634, "dur": 0.598,
    "args": {
      "External id": 2489630,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035838.557, "dur": 1.381,
    "args": {
      "External id": 2489631,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035839.212, "dur": 0.660,
    "args": {
      "External id": 2489632,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035840.181, "dur": 3.324,
    "args": {
      "External id": 2489633,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035842.944, "dur": 0.498,
    "args": {
      "External id": 2489634,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035843.756, "dur": 1.512,
    "args": {
      "External id": 2489635,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035844.654, "dur": 0.548,
    "args": {
      "External id": 2489636,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035845.488, "dur": 3.617,
    "args": {
      "External id": 2489637,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035846.728, "dur": 2.307,
    "args": {
      "External id": 2489638,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035851.182, "dur": 1.523,
    "args": {
      "External id": 2489639,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035851.917, "dur": 0.522,
    "args": {
      "External id": 2489640,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035852.922, "dur": 1.982,
    "args": {
      "External id": 2489641,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035853.939, "dur": 0.898,
    "args": {
      "External id": 2489642,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035855.142, "dur": 3.574,
    "args": {
      "External id": 2489643,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035858.032, "dur": 0.622,
    "args": {
      "External id": 2489644,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035858.961, "dur": 2.051,
    "args": {
      "External id": 2489645,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035860.230, "dur": 0.615,
    "args": {
      "External id": 2489646,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035861.257, "dur": 1.324,
    "args": {
      "External id": 2489647,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035861.893, "dur": 0.619,
    "args": {
      "External id": 2489648,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035862.856, "dur": 3.859,
    "args": {
      "External id": 2489649,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035865.842, "dur": 0.609,
    "args": {
      "External id": 2489650,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035866.959, "dur": 1.341,
    "args": {
      "External id": 2489651,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035867.657, "dur": 0.579,
    "args": {
      "External id": 2489652,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035868.644, "dur": 3.398,
    "args": {
      "External id": 2489653,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035869.827, "dur": 2.141,
    "args": {
      "External id": 2489654,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035874.235, "dur": 1.425,
    "args": {
      "External id": 2489655,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035874.867, "dur": 0.733,
    "args": {
      "External id": 2489656,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035875.884, "dur": 2.220,
    "args": {
      "External id": 2489657,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035877.329, "dur": 0.707,
    "args": {
      "External id": 2489658,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035878.348, "dur": 3.714,
    "args": {
      "External id": 2489659,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035881.460, "dur": 0.539,
    "args": {
      "External id": 2489660,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035882.338, "dur": 2.108,
    "args": {
      "External id": 2489661,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035883.712, "dur": 0.564,
    "args": {
      "External id": 2489662,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035884.709, "dur": 1.782,
    "args": {
      "External id": 2489663,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035885.529, "dur": 0.794,
    "args": {
      "External id": 2489664,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035886.727, "dur": 4.139,
    "args": {
      "External id": 2489665,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035890.216, "dur": 0.582,
    "args": {
      "External id": 2489666,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035891.289, "dur": 1.742,
    "args": {
      "External id": 2489667,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035891.947, "dur": 0.816,
    "args": {
      "External id": 2489668,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035893.252, "dur": 3.453,
    "args": {
      "External id": 2489669,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035894.479, "dur": 2.154,
    "args": {
      "External id": 2489670,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035898.876, "dur": 1.683,
    "args": {
      "External id": 2489671,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035899.610, "dur": 0.562,
    "args": {
      "External id": 2489672,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035900.971, "dur": 1.956,
    "args": {
      "External id": 2489673,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035902.185, "dur": 0.674,
    "args": {
      "External id": 2489674,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035903.170, "dur": 3.395,
    "args": {
      "External id": 2489675,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035905.898, "dur": 0.600,
    "args": {
      "External id": 2489676,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035906.806, "dur": 2.010,
    "args": {
      "External id": 2489677,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035908.127, "dur": 0.626,
    "args": {
      "External id": 2489678,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035909.060, "dur": 1.649,
    "args": {
      "External id": 2489679,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035910.031, "dur": 0.611,
    "args": {
      "External id": 2489680,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035910.984, "dur": 4.382,
    "args": {
      "External id": 2489681,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035914.456, "dur": 0.837,
    "args": {
      "External id": 2489682,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035915.611, "dur": 1.960,
    "args": {
      "External id": 2489683,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035916.471, "dur": 0.834,
    "args": {
      "External id": 2489684,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035917.795, "dur": 3.380,
    "args": {
      "External id": 2489685,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035918.781, "dur": 2.158,
    "args": {
      "External id": 2489686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035923.313, "dur": 1.676,
    "args": {
      "External id": 2489687,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035924.136, "dur": 0.580,
    "args": {
      "External id": 2489688,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035925.209, "dur": 2.308,
    "args": {
      "External id": 2489689,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035926.339, "dur": 0.916,
    "args": {
      "External id": 2489690,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035927.826, "dur": 3.432,
    "args": {
      "External id": 2489691,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035930.264, "dur": 0.925,
    "args": {
      "External id": 2489692,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035931.499, "dur": 1.980,
    "args": {
      "External id": 2489693,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035932.773, "dur": 0.541,
    "args": {
      "External id": 2489694,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035933.719, "dur": 1.616,
    "args": {
      "External id": 2489695,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035934.411, "dur": 0.855,
    "args": {
      "External id": 2489696,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035935.573, "dur": 3.529,
    "args": {
      "External id": 2489697,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035938.308, "dur": 0.734,
    "args": {
      "External id": 2489698,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035939.357, "dur": 1.394,
    "args": {
      "External id": 2489699,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035940.070, "dur": 0.619,
    "args": {
      "External id": 2489700,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035940.993, "dur": 3.506,
    "args": {
      "External id": 2489701,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035942.249, "dur": 2.177,
    "args": {
      "External id": 2489702,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035946.320, "dur": 1.345,
    "args": {
      "External id": 2489703,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035947.131, "dur": 0.474,
    "args": {
      "External id": 2489704,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035947.890, "dur": 2.037,
    "args": {
      "External id": 2489705,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035949.090, "dur": 0.769,
    "args": {
      "External id": 2489706,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035950.525, "dur": 3.674,
    "args": {
      "External id": 2489707,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035953.580, "dur": 0.556,
    "args": {
      "External id": 2489708,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035954.489, "dur": 2.043,
    "args": {
      "External id": 2489709,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035955.751, "dur": 0.719,
    "args": {
      "External id": 2489710,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035956.772, "dur": 1.415,
    "args": {
      "External id": 2489711,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035957.424, "dur": 0.694,
    "args": {
      "External id": 2489712,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035958.436, "dur": 3.797,
    "args": {
      "External id": 2489713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035961.316, "dur": 0.856,
    "args": {
      "External id": 2489714,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035962.473, "dur": 1.544,
    "args": {
      "External id": 2489715,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035963.125, "dur": 0.825,
    "args": {
      "External id": 2489716,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035964.241, "dur": 3.752,
    "args": {
      "External id": 2489717,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035965.515, "dur": 2.240,
    "args": {
      "External id": 2489718,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035970.347, "dur": 1.694,
    "args": {
      "External id": 2489719,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035971.022, "dur": 0.688,
    "args": {
      "External id": 2489720,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035972.451, "dur": 2.517,
    "args": {
      "External id": 2489721,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035973.758, "dur": 0.884,
    "args": {
      "External id": 2489722,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035975.215, "dur": 3.287,
    "args": {
      "External id": 2489723,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035977.827, "dur": 0.606,
    "args": {
      "External id": 2489724,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035978.746, "dur": 2.187,
    "args": {
      "External id": 2489725,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035980.095, "dur": 0.573,
    "args": {
      "External id": 2489726,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658035981.171, "dur": 25.860,
    "args": {
      "External id": 2489727,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036005.475, "dur": 1.100,
    "args": {
      "External id": 2489728,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036007.652, "dur": 3.930,
    "args": {
      "External id": 2489729,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036010.813, "dur": 0.705,
    "args": {
      "External id": 2489730,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036011.888, "dur": 3.035,
    "args": {
      "External id": 2489731,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036014.106, "dur": 0.754,
    "args": {
      "External id": 2489732,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036034.958, "dur": 141.765,
    "args": {
      "External id": 2489733,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036283.949, "dur": 138.668,
    "args": {
      "External id": 2489734,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036350.647, "dur": 51.314,
    "args": {
      "External id": 2489735,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036365.177, "dur": 0.966,
    "args": {
      "External id": 2489736,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Redistribute", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036912.476, "dur": 1020.501,
    "args": {
      "External id": 2489737,"Sequence number": 24740656, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "False"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036976.730, "dur": 142.062,
    "args": {
      "External id": 2489738,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658036981.032, "dur": 29.277,
    "args": {
      "External id": 2489739,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037068.301, "dur": 0.974,
    "args": {
      "External id": 2489740,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::all_reduce", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037149.431, "dur": 458.293,
    "args": {
      "External id": 2489741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["float", "", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037155.364, "dur": 47.290,
    "args": {
      "External id": 2489742,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037158.920, "dur": 8.499,
    "args": {
      "External id": 2489743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "0"], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037163.030, "dur": 3.648,
    "args": {
      "External id": 2489744,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037168.770, "dur": 33.281,
    "args": {
      "External id": 2489745,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::allreduce_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037212.608, "dur": 390.875,
    "args": {
      "External id": 2489746,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "-1"], "Input type": ["TensorList", "", "", "", "Scalar"], "Input Strides": [[[]], [], [], [], []], "Input Dims": [[[]], [], [], [], []], "Ev Idx": 11305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037245.805, "dur": 350.076,
    "args": {
      "External id": 2489747,"Record function id": 0, "Collective name": "allreduce", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 7, "Input Strides": [[[]], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "7", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 1, "Process Group Name": "0", "Input type": ["TensorList", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[[]], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 11306, "In msg nelems": 1
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:all_reduce", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037265.338, "dur": 323.943,
    "args": {
      "External id": 2489748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037678.953, "dur": 215.111,
    "args": {
      "External id": 2489749,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::wait_tensor", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037779.884, "dur": 33.152,
    "args": {
      "External id": 2489750,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037799.776, "dur": 4.123,
    "args": {
      "External id": 2489751,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "7", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 11310, "In msg nelems": 0, "Rank": 7, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037843.003, "dur": 45.043,
    "args": {
      "External id": 2489752,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037845.583, "dur": 1.012,
    "args": {
      "External id": 2489753,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037847.973, "dur": 0.767,
    "args": {
      "External id": 2489754,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_ToTorchTensor", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037950.661, "dur": 23.346,
    "args": {
      "External id": 2489755,"Sequence number": 24740657, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037962.006, "dur": 8.658,
    "args": {
      "External id": 2489756,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336760, "tid": 1336760,
    "ts": 1514658037964.327, "dur": 6.154,
    "args": {
      "External id": 2489757,"Record function id": 0, "Concrete Inputs": ["", "[]"], "Input type": ["float", "ScalarList"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038407.094, "dur": 39.137,
    "args": {
      "External id": 2489758,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "double", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reciprocal", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038455.858, "dur": 29.711,
    "args": {
      "External id": 2489759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mul", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038494.693, "dur": 27.236,
    "args": {
      "External id": 2489760,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "double"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clamp", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038551.195, "dur": 28.899,
    "args": {
      "External id": 2489761,"Record function id": 0, "Concrete Inputs": ["", "", "1."], "Input type": ["float", "", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038554.269, "dur": 0.622,
    "args": {
      "External id": 2489762,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038600.417, "dur": 0.408,
    "args": {
      "External id": 2489763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 11322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658038724.428, "dur": 858.989,
    "args": {
      "External id": 2489764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039263.946, "dur": 287.069,
    "args": {
      "External id": 2489765,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isnan", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039631.903, "dur": 34.914,
    "args": {
      "External id": 2489766,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039636.334, "dur": 30.098,
    "args": {
      "External id": 2489767,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039672.337, "dur": 173.425,
    "args": {
      "External id": 2489768,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039674.107, "dur": 171.469,
    "args": {
      "External id": 2489769,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039675.954, "dur": 169.278,
    "args": {
      "External id": 2489770,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isinf", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039850.824, "dur": 61.480,
    "args": {
      "External id": 2489771,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039853.484, "dur": 34.037,
    "args": {
      "External id": 2489772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039860.728, "dur": 2.915,
    "args": {
      "External id": 2489773,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039865.399, "dur": 21.723,
    "args": {
      "External id": 2489774,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], [1]], "Input Dims": [[], [0]], "Ev Idx": 11333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039870.204, "dur": 2.511,
    "args": {
      "External id": 2489775,"Record function id": 0, "Concrete Inputs": ["", "[]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 11334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039889.390, "dur": 21.868,
    "args": {
      "External id": 2489776,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039914.219, "dur": 36.797,
    "args": {
      "External id": 2489777,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039917.257, "dur": 33.615,
    "args": {
      "External id": 2489778,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336760, "tid": 1336760,
    "ts": 1514658039918.267, "dur": 32.381,
    "args": {
      "External id": 2489779,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11338
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#OptimizersContainer.step", "pid": 1336760, "tid": 1336760,
    "ts": 1514658040018.234, "dur": 5954.117,
    "args": {
      "External id": 2489780,"Record function id": 0, "Ev Idx": 11339
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#AdamW.step", "pid": 1336760, "tid": 1336760,
    "ts": 1514658040055.635, "dur": 5895.175,
    "args": {
      "External id": 2489781,"Record function id": 0, "Ev Idx": 11340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_add_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041450.516, "dur": 313.685,
    "args": {
      "External id": 2489782,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041469.866, "dur": 1.301,
    "args": {
      "External id": 2489783,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041472.150, "dur": 0.289,
    "args": {
      "External id": 2489784,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041472.913, "dur": 0.376,
    "args": {
      "External id": 2489785,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041473.874, "dur": 0.107,
    "args": {
      "External id": 2489786,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041474.678, "dur": 0.055,
    "args": {
      "External id": 2489787,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041475.117, "dur": 0.054,
    "args": {
      "External id": 2489788,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041475.784, "dur": 0.068,
    "args": {
      "External id": 2489789,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041476.699, "dur": 0.061,
    "args": {
      "External id": 2489790,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041477.541, "dur": 0.053,
    "args": {
      "External id": 2489791,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041478.180, "dur": 0.237,
    "args": {
      "External id": 2489792,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041479.189, "dur": 0.093,
    "args": {
      "External id": 2489793,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041479.635, "dur": 0.070,
    "args": {
      "External id": 2489794,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041480.334, "dur": 0.062,
    "args": {
      "External id": 2489795,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041480.789, "dur": 0.070,
    "args": {
      "External id": 2489796,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041481.498, "dur": 0.057,
    "args": {
      "External id": 2489797,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041481.905, "dur": 0.067,
    "args": {
      "External id": 2489798,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041482.322, "dur": 0.064,
    "args": {
      "External id": 2489799,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041482.754, "dur": 0.055,
    "args": {
      "External id": 2489800,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041483.197, "dur": 0.059,
    "args": {
      "External id": 2489801,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041483.607, "dur": 0.245,
    "args": {
      "External id": 2489802,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041484.265, "dur": 0.067,
    "args": {
      "External id": 2489803,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041484.658, "dur": 0.071,
    "args": {
      "External id": 2489804,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041485.280, "dur": 0.067,
    "args": {
      "External id": 2489805,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041485.724, "dur": 0.287,
    "args": {
      "External id": 2489806,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041486.450, "dur": 0.101,
    "args": {
      "External id": 2489807,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041487.106, "dur": 0.094,
    "args": {
      "External id": 2489808,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041487.882, "dur": 0.300,
    "args": {
      "External id": 2489809,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041488.709, "dur": 0.260,
    "args": {
      "External id": 2489810,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041489.442, "dur": 0.056,
    "args": {
      "External id": 2489811,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041489.896, "dur": 0.060,
    "args": {
      "External id": 2489812,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041490.446, "dur": 0.065,
    "args": {
      "External id": 2489813,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041491.129, "dur": 0.062,
    "args": {
      "External id": 2489814,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041491.533, "dur": 0.069,
    "args": {
      "External id": 2489815,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041491.998, "dur": 0.064,
    "args": {
      "External id": 2489816,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041492.915, "dur": 0.063,
    "args": {
      "External id": 2489817,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041493.364, "dur": 0.071,
    "args": {
      "External id": 2489818,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041493.869, "dur": 0.062,
    "args": {
      "External id": 2489819,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041494.508, "dur": 0.065,
    "args": {
      "External id": 2489820,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041495.019, "dur": 0.068,
    "args": {
      "External id": 2489821,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041495.769, "dur": 0.065,
    "args": {
      "External id": 2489822,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041496.235, "dur": 0.061,
    "args": {
      "External id": 2489823,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041496.682, "dur": 0.058,
    "args": {
      "External id": 2489824,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041497.100, "dur": 0.066,
    "args": {
      "External id": 2489825,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041497.648, "dur": 0.057,
    "args": {
      "External id": 2489826,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041498.164, "dur": 0.069,
    "args": {
      "External id": 2489827,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041498.717, "dur": 0.060,
    "args": {
      "External id": 2489828,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041499.316, "dur": 0.066,
    "args": {
      "External id": 2489829,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041499.849, "dur": 0.061,
    "args": {
      "External id": 2489830,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041500.286, "dur": 0.300,
    "args": {
      "External id": 2489831,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041501.035, "dur": 0.239,
    "args": {
      "External id": 2489832,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041501.648, "dur": 0.115,
    "args": {
      "External id": 2489833,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041502.237, "dur": 0.294,
    "args": {
      "External id": 2489834,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041503.011, "dur": 0.060,
    "args": {
      "External id": 2489835,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041503.432, "dur": 0.065,
    "args": {
      "External id": 2489836,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041504.011, "dur": 0.065,
    "args": {
      "External id": 2489837,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041504.480, "dur": 0.064,
    "args": {
      "External id": 2489838,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041505.061, "dur": 0.066,
    "args": {
      "External id": 2489839,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041505.509, "dur": 0.064,
    "args": {
      "External id": 2489840,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041506.041, "dur": 0.060,
    "args": {
      "External id": 2489841,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041506.465, "dur": 0.057,
    "args": {
      "External id": 2489842,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041507.121, "dur": 0.065,
    "args": {
      "External id": 2489843,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041507.625, "dur": 0.064,
    "args": {
      "External id": 2489844,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041508.057, "dur": 0.055,
    "args": {
      "External id": 2489845,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041508.496, "dur": 0.054,
    "args": {
      "External id": 2489846,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041509.021, "dur": 0.070,
    "args": {
      "External id": 2489847,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041509.591, "dur": 0.066,
    "args": {
      "External id": 2489848,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041510.164, "dur": 0.062,
    "args": {
      "External id": 2489849,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041510.524, "dur": 0.064,
    "args": {
      "External id": 2489850,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041511.079, "dur": 0.065,
    "args": {
      "External id": 2489851,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041511.584, "dur": 0.067,
    "args": {
      "External id": 2489852,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041512.135, "dur": 0.065,
    "args": {
      "External id": 2489853,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041512.548, "dur": 0.067,
    "args": {
      "External id": 2489854,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041512.916, "dur": 0.061,
    "args": {
      "External id": 2489855,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041513.433, "dur": 0.064,
    "args": {
      "External id": 2489856,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041550.712, "dur": 0.410,
    "args": {
      "External id": 2489857,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041552.520, "dur": 0.055,
    "args": {
      "External id": 2489858,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041553.177, "dur": 0.071,
    "args": {
      "External id": 2489859,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041553.942, "dur": 0.055,
    "args": {
      "External id": 2489860,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041554.327, "dur": 0.060,
    "args": {
      "External id": 2489861,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041554.755, "dur": 0.066,
    "args": {
      "External id": 2489862,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041555.199, "dur": 0.062,
    "args": {
      "External id": 2489863,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041555.632, "dur": 0.069,
    "args": {
      "External id": 2489864,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041556.041, "dur": 0.067,
    "args": {
      "External id": 2489865,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041556.741, "dur": 0.067,
    "args": {
      "External id": 2489866,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041557.428, "dur": 0.054,
    "args": {
      "External id": 2489867,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041557.868, "dur": 0.056,
    "args": {
      "External id": 2489868,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041558.486, "dur": 0.059,
    "args": {
      "External id": 2489869,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041559.191, "dur": 0.061,
    "args": {
      "External id": 2489870,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041559.675, "dur": 0.049,
    "args": {
      "External id": 2489871,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041560.121, "dur": 0.060,
    "args": {
      "External id": 2489872,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041560.729, "dur": 0.065,
    "args": {
      "External id": 2489873,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041561.197, "dur": 0.057,
    "args": {
      "External id": 2489874,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041561.814, "dur": 0.060,
    "args": {
      "External id": 2489875,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041562.193, "dur": 0.058,
    "args": {
      "External id": 2489876,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041562.605, "dur": 0.069,
    "args": {
      "External id": 2489877,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041562.955, "dur": 0.066,
    "args": {
      "External id": 2489878,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041563.388, "dur": 0.066,
    "args": {
      "External id": 2489879,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041563.822, "dur": 0.064,
    "args": {
      "External id": 2489880,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041564.400, "dur": 0.056,
    "args": {
      "External id": 2489881,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041564.775, "dur": 0.066,
    "args": {
      "External id": 2489882,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041565.364, "dur": 0.065,
    "args": {
      "External id": 2489883,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041565.699, "dur": 0.070,
    "args": {
      "External id": 2489884,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041566.216, "dur": 0.068,
    "args": {
      "External id": 2489885,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041566.935, "dur": 0.071,
    "args": {
      "External id": 2489886,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041567.427, "dur": 0.070,
    "args": {
      "External id": 2489887,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041568.000, "dur": 0.066,
    "args": {
      "External id": 2489888,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041568.572, "dur": 0.069,
    "args": {
      "External id": 2489889,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041569.232, "dur": 0.071,
    "args": {
      "External id": 2489890,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041569.633, "dur": 0.074,
    "args": {
      "External id": 2489891,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041570.209, "dur": 0.071,
    "args": {
      "External id": 2489892,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041570.867, "dur": 0.054,
    "args": {
      "External id": 2489893,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041571.409, "dur": 0.068,
    "args": {
      "External id": 2489894,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041572.004, "dur": 0.069,
    "args": {
      "External id": 2489895,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041572.564, "dur": 0.067,
    "args": {
      "External id": 2489896,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041573.150, "dur": 0.053,
    "args": {
      "External id": 2489897,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041573.601, "dur": 0.095,
    "args": {
      "External id": 2489898,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041574.040, "dur": 0.071,
    "args": {
      "External id": 2489899,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041574.449, "dur": 0.065,
    "args": {
      "External id": 2489900,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041575.010, "dur": 0.070,
    "args": {
      "External id": 2489901,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041575.456, "dur": 0.068,
    "args": {
      "External id": 2489902,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041575.867, "dur": 0.066,
    "args": {
      "External id": 2489903,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041576.304, "dur": 0.063,
    "args": {
      "External id": 2489904,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041576.670, "dur": 0.069,
    "args": {
      "External id": 2489905,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041577.255, "dur": 0.072,
    "args": {
      "External id": 2489906,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041577.656, "dur": 0.070,
    "args": {
      "External id": 2489907,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041578.044, "dur": 0.090,
    "args": {
      "External id": 2489908,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041578.420, "dur": 0.065,
    "args": {
      "External id": 2489909,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041586.762, "dur": 0.088,
    "args": {
      "External id": 2489910,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041587.608, "dur": 0.067,
    "args": {
      "External id": 2489911,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041588.166, "dur": 0.071,
    "args": {
      "External id": 2489912,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041588.697, "dur": 0.058,
    "args": {
      "External id": 2489913,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041589.013, "dur": 0.073,
    "args": {
      "External id": 2489914,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041589.442, "dur": 0.063,
    "args": {
      "External id": 2489915,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041589.847, "dur": 0.057,
    "args": {
      "External id": 2489916,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041590.466, "dur": 0.066,
    "args": {
      "External id": 2489917,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041590.847, "dur": 0.068,
    "args": {
      "External id": 2489918,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041591.273, "dur": 0.067,
    "args": {
      "External id": 2489919,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041591.664, "dur": 0.065,
    "args": {
      "External id": 2489920,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041592.045, "dur": 0.072,
    "args": {
      "External id": 2489921,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041592.781, "dur": 0.057,
    "args": {
      "External id": 2489922,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041593.338, "dur": 0.070,
    "args": {
      "External id": 2489923,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041593.892, "dur": 0.069,
    "args": {
      "External id": 2489924,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041594.520, "dur": 0.069,
    "args": {
      "External id": 2489925,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041595.039, "dur": 0.067,
    "args": {
      "External id": 2489926,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041595.750, "dur": 0.069,
    "args": {
      "External id": 2489927,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041596.151, "dur": 0.068,
    "args": {
      "External id": 2489928,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041596.784, "dur": 0.067,
    "args": {
      "External id": 2489929,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041597.161, "dur": 0.071,
    "args": {
      "External id": 2489930,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041597.530, "dur": 0.069,
    "args": {
      "External id": 2489931,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041598.059, "dur": 0.074,
    "args": {
      "External id": 2489932,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041598.415, "dur": 0.067,
    "args": {
      "External id": 2489933,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041599.001, "dur": 0.069,
    "args": {
      "External id": 2489934,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041599.500, "dur": 0.068,
    "args": {
      "External id": 2489935,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041599.911, "dur": 0.067,
    "args": {
      "External id": 2489936,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041600.285, "dur": 0.068,
    "args": {
      "External id": 2489937,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041600.778, "dur": 0.066,
    "args": {
      "External id": 2489938,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041601.203, "dur": 0.068,
    "args": {
      "External id": 2489939,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041601.711, "dur": 0.070,
    "args": {
      "External id": 2489940,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041602.281, "dur": 0.054,
    "args": {
      "External id": 2489941,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041602.984, "dur": 0.065,
    "args": {
      "External id": 2489942,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041603.474, "dur": 0.058,
    "args": {
      "External id": 2489943,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041603.979, "dur": 0.060,
    "args": {
      "External id": 2489944,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041604.308, "dur": 0.056,
    "args": {
      "External id": 2489945,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041604.699, "dur": 0.059,
    "args": {
      "External id": 2489946,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041605.193, "dur": 0.055,
    "args": {
      "External id": 2489947,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041605.607, "dur": 0.059,
    "args": {
      "External id": 2489948,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041606.149, "dur": 0.057,
    "args": {
      "External id": 2489949,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041606.754, "dur": 0.060,
    "args": {
      "External id": 2489950,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041607.382, "dur": 0.057,
    "args": {
      "External id": 2489951,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041607.951, "dur": 0.054,
    "args": {
      "External id": 2489952,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041608.322, "dur": 0.072,
    "args": {
      "External id": 2489953,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041608.853, "dur": 0.059,
    "args": {
      "External id": 2489954,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041609.332, "dur": 0.051,
    "args": {
      "External id": 2489955,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041609.830, "dur": 0.063,
    "args": {
      "External id": 2489956,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041610.221, "dur": 0.064,
    "args": {
      "External id": 2489957,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041610.752, "dur": 0.088,
    "args": {
      "External id": 2489958,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041611.188, "dur": 0.066,
    "args": {
      "External id": 2489959,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041611.922, "dur": 0.066,
    "args": {
      "External id": 2489960,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041612.498, "dur": 0.067,
    "args": {
      "External id": 2489961,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041613.018, "dur": 0.062,
    "args": {
      "External id": 2489962,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041613.434, "dur": 0.058,
    "args": {
      "External id": 2489963,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041613.987, "dur": 0.068,
    "args": {
      "External id": 2489964,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041614.351, "dur": 0.063,
    "args": {
      "External id": 2489965,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041614.788, "dur": 0.067,
    "args": {
      "External id": 2489966,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041615.282, "dur": 0.065,
    "args": {
      "External id": 2489967,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041615.678, "dur": 0.067,
    "args": {
      "External id": 2489968,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041616.185, "dur": 0.070,
    "args": {
      "External id": 2489969,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041616.757, "dur": 0.059,
    "args": {
      "External id": 2489970,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041617.216, "dur": 0.059,
    "args": {
      "External id": 2489971,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041617.651, "dur": 0.062,
    "args": {
      "External id": 2489972,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041618.125, "dur": 0.060,
    "args": {
      "External id": 2489973,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041618.512, "dur": 0.059,
    "args": {
      "External id": 2489974,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041619.012, "dur": 0.055,
    "args": {
      "External id": 2489975,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041619.439, "dur": 0.049,
    "args": {
      "External id": 2489976,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041620.223, "dur": 0.059,
    "args": {
      "External id": 2489977,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041620.739, "dur": 0.060,
    "args": {
      "External id": 2489978,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041621.351, "dur": 0.054,
    "args": {
      "External id": 2489979,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041621.677, "dur": 0.062,
    "args": {
      "External id": 2489980,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041622.062, "dur": 0.066,
    "args": {
      "External id": 2489981,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041622.695, "dur": 0.066,
    "args": {
      "External id": 2489982,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041623.079, "dur": 0.066,
    "args": {
      "External id": 2489983,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041623.782, "dur": 0.063,
    "args": {
      "External id": 2489984,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041624.175, "dur": 0.063,
    "args": {
      "External id": 2489985,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041624.681, "dur": 0.069,
    "args": {
      "External id": 2489986,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041625.259, "dur": 0.066,
    "args": {
      "External id": 2489987,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041625.788, "dur": 0.066,
    "args": {
      "External id": 2489988,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041626.185, "dur": 0.066,
    "args": {
      "External id": 2489989,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041626.716, "dur": 0.068,
    "args": {
      "External id": 2489990,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041627.220, "dur": 0.069,
    "args": {
      "External id": 2489991,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041627.742, "dur": 0.064,
    "args": {
      "External id": 2489992,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041628.289, "dur": 0.066,
    "args": {
      "External id": 2489993,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041628.962, "dur": 0.074,
    "args": {
      "External id": 2489994,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041629.364, "dur": 0.074,
    "args": {
      "External id": 2489995,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041630.002, "dur": 0.066,
    "args": {
      "External id": 2489996,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041630.311, "dur": 0.055,
    "args": {
      "External id": 2489997,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041631.184, "dur": 0.065,
    "args": {
      "External id": 2489998,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041631.492, "dur": 0.053,
    "args": {
      "External id": 2489999,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041632.215, "dur": 0.067,
    "args": {
      "External id": 2490000,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041632.524, "dur": 0.054,
    "args": {
      "External id": 2490001,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041633.266, "dur": 0.065,
    "args": {
      "External id": 2490002,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041633.629, "dur": 0.068,
    "args": {
      "External id": 2490003,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041634.245, "dur": 0.068,
    "args": {
      "External id": 2490004,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041634.552, "dur": 0.054,
    "args": {
      "External id": 2490005,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041635.371, "dur": 0.066,
    "args": {
      "External id": 2490006,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041635.683, "dur": 0.054,
    "args": {
      "External id": 2490007,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041636.411, "dur": 0.060,
    "args": {
      "External id": 2490008,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041636.913, "dur": 0.071,
    "args": {
      "External id": 2490009,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041637.611, "dur": 0.060,
    "args": {
      "External id": 2490010,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041637.934, "dur": 0.051,
    "args": {
      "External id": 2490011,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041638.831, "dur": 0.053,
    "args": {
      "External id": 2490012,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041639.130, "dur": 0.054,
    "args": {
      "External id": 2490013,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041639.952, "dur": 0.065,
    "args": {
      "External id": 2490014,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041640.261, "dur": 0.054,
    "args": {
      "External id": 2490015,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041640.912, "dur": 0.067,
    "args": {
      "External id": 2490016,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041641.224, "dur": 0.053,
    "args": {
      "External id": 2490017,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041641.896, "dur": 0.064,
    "args": {
      "External id": 2490018,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041642.208, "dur": 0.054,
    "args": {
      "External id": 2490019,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041643.062, "dur": 0.057,
    "args": {
      "External id": 2490020,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041643.362, "dur": 0.049,
    "args": {
      "External id": 2490021,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041644.164, "dur": 0.066,
    "args": {
      "External id": 2490022,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041644.514, "dur": 0.051,
    "args": {
      "External id": 2490023,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041645.238, "dur": 0.063,
    "args": {
      "External id": 2490024,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041645.548, "dur": 0.056,
    "args": {
      "External id": 2490025,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041646.116, "dur": 0.068,
    "args": {
      "External id": 2490026,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041646.439, "dur": 0.053,
    "args": {
      "External id": 2490027,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041647.004, "dur": 0.063,
    "args": {
      "External id": 2490028,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041647.328, "dur": 0.052,
    "args": {
      "External id": 2490029,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041648.195, "dur": 0.068,
    "args": {
      "External id": 2490030,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041648.522, "dur": 0.055,
    "args": {
      "External id": 2490031,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041649.349, "dur": 0.068,
    "args": {
      "External id": 2490032,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041649.676, "dur": 0.049,
    "args": {
      "External id": 2490033,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041650.280, "dur": 0.068,
    "args": {
      "External id": 2490034,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041650.605, "dur": 0.053,
    "args": {
      "External id": 2490035,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041651.410, "dur": 0.066,
    "args": {
      "External id": 2490036,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041651.730, "dur": 0.053,
    "args": {
      "External id": 2490037,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041652.482, "dur": 0.050,
    "args": {
      "External id": 2490038,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041652.791, "dur": 0.052,
    "args": {
      "External id": 2490039,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041653.538, "dur": 0.067,
    "args": {
      "External id": 2490040,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041653.861, "dur": 0.054,
    "args": {
      "External id": 2490041,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041654.542, "dur": 0.067,
    "args": {
      "External id": 2490042,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041654.863, "dur": 0.052,
    "args": {
      "External id": 2490043,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041655.568, "dur": 0.062,
    "args": {
      "External id": 2490044,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041655.888, "dur": 0.054,
    "args": {
      "External id": 2490045,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041656.656, "dur": 0.054,
    "args": {
      "External id": 2490046,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041656.970, "dur": 0.053,
    "args": {
      "External id": 2490047,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041657.730, "dur": 0.061,
    "args": {
      "External id": 2490048,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041658.045, "dur": 0.051,
    "args": {
      "External id": 2490049,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041658.750, "dur": 0.060,
    "args": {
      "External id": 2490050,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041659.067, "dur": 0.049,
    "args": {
      "External id": 2490051,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041659.797, "dur": 0.071,
    "args": {
      "External id": 2490052,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041660.120, "dur": 0.053,
    "args": {
      "External id": 2490053,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041661.063, "dur": 0.066,
    "args": {
      "External id": 2490054,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041661.385, "dur": 0.051,
    "args": {
      "External id": 2490055,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041662.077, "dur": 0.069,
    "args": {
      "External id": 2490056,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041662.425, "dur": 0.056,
    "args": {
      "External id": 2490057,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041663.345, "dur": 0.058,
    "args": {
      "External id": 2490058,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041663.656, "dur": 0.056,
    "args": {
      "External id": 2490059,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041664.405, "dur": 0.072,
    "args": {
      "External id": 2490060,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041664.734, "dur": 0.055,
    "args": {
      "External id": 2490061,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041665.336, "dur": 0.067,
    "args": {
      "External id": 2490062,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041665.653, "dur": 0.051,
    "args": {
      "External id": 2490063,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041666.463, "dur": 0.069,
    "args": {
      "External id": 2490064,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041666.787, "dur": 0.054,
    "args": {
      "External id": 2490065,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041667.370, "dur": 0.072,
    "args": {
      "External id": 2490066,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041667.697, "dur": 0.055,
    "args": {
      "External id": 2490067,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041668.245, "dur": 0.069,
    "args": {
      "External id": 2490068,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041668.568, "dur": 0.053,
    "args": {
      "External id": 2490069,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041669.148, "dur": 0.068,
    "args": {
      "External id": 2490070,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041669.474, "dur": 0.051,
    "args": {
      "External id": 2490071,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041670.313, "dur": 0.064,
    "args": {
      "External id": 2490072,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336760, "tid": 1336760,
    "ts": 1514658041670.636, "dur": 0.054,
    "args": {
      "External id": 2490073,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658042306.685, "dur": 3549.392,
    "args": {
      "External id": 2490074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336760, "tid": 1336760,
    "ts": 1514658045350.163, "dur": 344.095,
    "args": {
      "External id": 2490075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11634
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 0,
    "args": {
      "name": "python3.12"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 0,
    "args": {
      "sort_index": 1336760
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1381209,
    "args": {
      "name": "thread 1381209 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1381209,
    "args": {
      "sort_index": 1381209
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1381209,
    "args": {
      "name": "thread 1381209 (pt_autograd_7)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1381209,
    "args": {
      "sort_index": 1381209
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1336760,
    "args": {
      "name": "thread 1336760 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655938197.931, "pid": 1336760, "tid": 1336760,
    "args": {
      "sort_index": 1336760
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1514655938129.312, "dur": 2109485.121,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655938129.312,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1514655938129.312
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1514658123176.054
  }
  ],
  "traceName": "exp/mtp.1B.batch16.seqlen4096.context4096.warmup2000.update1.steps200000.lr2e-4.cosine/profile_trace/iteration_102912/rank7_trace.json",
  "displayTimeUnit": "ms",
  "baseTimeNanoseconds": 1751410836000000000
}